Scan all images under a namespace for Docker remote registries (#4514)

* Scan all images under a namespace for Docker remote registries

* improvements

* Resolved charlie's comment

* added logs

* formatted error messages

Author: kashifkhan0771

main.go

@@ -193,10 +193,12 @@ var (
 	circleCiScan      = cli.Command("circleci", "Scan CircleCI")
 	circleCiScanToken = circleCiScan.Flag("token", "CircleCI token. Can also be provided with environment variable").Envar("CIRCLECI_TOKEN").Required().String()
 
-	dockerScan         = cli.Command("docker", "Scan Docker Image")
-	dockerScanImages   = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Required().Strings()
-	dockerScanToken    = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
-	dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
+	dockerScan              = cli.Command("docker", "Scan Docker Image")
+	dockerScanImages        = dockerScan.Flag("image", "Docker image to scan. Use the file:// prefix to point to a local tarball, the docker:// prefix to point to the docker daemon, otherwise an image registry is assumed.").Strings()
+	dockerScanToken         = dockerScan.Flag("token", "Docker bearer token. Can also be provided with environment variable").Envar("DOCKER_TOKEN").String()
+	dockerExcludePaths      = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
+	dockerScanNamespace     = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String()
+	dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String()
 
 	travisCiScan      = cli.Command("travisci", "Scan TravisCI")
 	travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
@@ -934,11 +936,25 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			refs = []sources.JobProgressRef{ref}
 		}
 	case dockerScan.FullCommand():
+		if *dockerScanImages != nil && *dockerScanNamespace != "" {
+			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time")
+		}
+
+		if *dockerScanImages == nil && *dockerScanNamespace == "" {
+			return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required")
+		}
+
+		if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" {
+			return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace")
+		}
+
 		cfg := sources.DockerConfig{
 			BearerToken:       *dockerScanToken,
 			Images:            *dockerScanImages,
 			UseDockerKeychain: *dockerScanToken == "",
 			ExcludePaths:      strings.Split(*dockerExcludePaths, ","),
+			Namespace:         *dockerScanNamespace,
+			RegistryToken:     *dockerScanRegistryToken,
 		}
 		if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)

pkg/engine/docker.go

@@ -15,8 +15,10 @@ import (
 // ScanDocker scans a given docker connection.
 func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (sources.JobProgressRef, error) {
 	connection := &sourcespb.Docker{
-		Images:       c.Images,
-		ExcludePaths: c.ExcludePaths,
+		Images:        c.Images,
+		ExcludePaths:  c.ExcludePaths,
+		Namespace:     c.Namespace,
+		RegistryToken: c.RegistryToken,
 	}
 
 	switch {

pkg/pb/sourcespb/sources.pb.go

@@ -32,14 +32,15 @@ Docker is a containerization platform that packages applications and their depen
 - **Authentication Support**: Multiple authentication methods for private registries
 - **File Exclusion**: Configure patterns to skip specific files or directories
 - **Size Limits**: Automatically skips files exceeding 50MB to optimize performance
+- **Scan All Images Under a Namespace**: Enables automatic discovery and scanning of all container images under a specified namespace (organization or user) in supported registries such as Docker Hub, Quay, and GHCR. Users no longer need to manually list or specify individual image names. The system retrieves all public images within the namespace, and if a valid registry token is provided includes private images as well. This allows for large-scale, automated scanning across all repositories within an organization.
 
 ## Configuration
 
 ### Connection Types
 
 The Docker source supports several image reference formats:
 
-```go
+```text
 // Remote registry (default)
 "nginx:latest"
 "myregistry.com/myapp:v1.0.0"
@@ -51,6 +52,7 @@ The Docker source supports several image reference formats:
 // Tarball file
 "file:///path/to/image.tar"
 ```
+
 ### Authentication Methods
 
 #### 1. Unauthenticated (Public Images)
@@ -159,6 +161,47 @@ docker login quay.io
 cat ~/.docker/config.json
 ```
 
+---
+
+### Namespace Scanning (This feature is currently in beta version and under testing)
+
+To scan **all images** under a namespace (organization or user):
+
+**CLI Usage:**
+```bash
+# If no registry prefix is provided, Docker Hub is used by default
+trufflehog docker --namespace myorg
+
+# For other registries, include the registry prefix (e.g., quay.io, ghcr.io)
+trufflehog docker --namespace quay.io/my_namespace
+```
+
+To include private images within that namespace:
+```bash
+trufflehog docker --namespace myorg --registry-token <access_token>
+```
+
+**YAML Configuration:**
+```yaml
+sources:
+  - type: docker
+    name: org-scan
+    docker:
+      namespace: myorg
+      registry_token: "ghp_xxxxxxxxxxxxxxxxxxxx"
+```
+
+Supported registries:
+- Docker Hub (`docker.io`)
+- Quay (`quay.io`)
+- GitHub Container Registry (`ghcr.io`)
+
+This mode automatically enumerates all repositories within the specified namespace before scanning.
+
+Note: According to the GHCR documentation, only GitHub Classic Personal Access Tokens (PATs) are currently supported for accessing container packages - including public ones.
+Source: [GitHub Roadmap Issue #558](https://github.com/github/roadmap/issues/558)
+
+---
 
 ### File Exclusion
 
@@ -200,6 +243,18 @@ trufflehog docker --image myregistry.com/private-image:latest --exclude-paths **
 trufflehog docker --image nginx:latest
 ```
 
+### Scanning All Images Under a Namespace (Beta Version)
+
+```bash
+trufflehog docker --namespace trufflesecurity
+```
+
+Including private images:
+
+```bash
+trufflehog docker --namespace trufflesecurity --registry-token ghp_xxxxxxxxxxxxxxxxxxxx
+```
+
 ### Scanning Multiple Images
 
 ```bash
@@ -215,10 +270,7 @@ trufflehog docker --image docker://myapp:local
 ### Scanning a Tarball
 
 ```bash
-# First, save an image to a tarball
 docker save myapp:latest -o myapp.tar
-
-# Then scan it
 trufflehog docker --image file:///path/to/myapp.tar
 ```
 
@@ -231,15 +283,14 @@ trufflehog docker --image my-registry.io/private-app:v1.0.0
 
 ## Testing Results
 
-### Integration Test Results
-
 | Test Case | Status | Command/Configuration | Registry URL | Notes |
 |-----------|--------|----------------------|--------------|-------|
 | Scan remote image on DockerHub | ✅ Success | `--image <image_name>` | https://hub.docker.com/ | Public images work without authentication |
 | Scan specific tag of image on DockerHub | ✅ Success | `--image <image_name>:<tag_name>` | https://hub.docker.com/ | Tag specification working correctly |
+| Scan all images under namespace | In Progress | `--namespace <namespace>` | DockerHub, Quay, GHCR | Automatically discovers all public images |
 | Scan remote image on Quay.io | ✅ Success | `--image quay.io/prometheus/prometheus` | https://quay.io/search | Public Quay.io registry supported |
 | Scan multiple images | ✅ Success | `--image <image_name> --image <image_name>` | Multiple registries | Sequential scanning of multiple images |
-| Scan remote image on DockerHub with token | ✅ Success | Generate token using username and password | https://hub.docker.com/ | Basic auth with PAT working |
+| Scan remote image on DockerHub with token | ✅ Success | `--token <token>`(Generate token using username and password) | https://hub.docker.com/ | Authenticated scanning for private repos |
 | Scan private image on Quay | ⏸️ Halted | N/A | https://quay.io/ | RedHat requires paid account for private repos |
 | Scan private image on GHCR | ✅ Success | `--image ghcr.io/<image_name>` | https://github.com/packages | GitHub Container Registry |
 
@@ -248,23 +299,19 @@ trufflehog docker --image my-registry.io/private-app:v1.0.0
 ### Common Issues
 
 **Issue**: Authentication failures with private registries
-
-**Solution**: Ensure credentials are correct and have pull permissions. Use `docker login` first when using Docker Keychain method.
+**Solution**: Ensure credentials are correct and have pull permissions. Use `docker login` first when using Docker Keychain.
 
 ---
 
 **Issue**: Out of memory errors with large images
-
 **Solution**: Reduce concurrency or scan smaller images. Consider increasing available memory.
 
 ---
 
 **Issue**: Slow scanning performance
-
 **Solution**: Enable concurrent processing, use local daemon instead of remote registry, or exclude unnecessary directories.
 
 ---
 
 **Issue**: Files not being scanned
-
 **Solution**: Check exclude patterns and file size limits. Verify files are under 50MB.

pkg/pb/sourcespb/sources.pb.validate.go

@@ -62,7 +62,7 @@ func (s *Source) JobID() sources.JobID {
 }
 
 // Init initializes the source.
-func (s *Source) Init(_ context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
+func (s *Source) Init(ctx context.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
 	s.name = name
 	s.sourceId = sourceId
 	s.jobId = jobId
@@ -119,6 +119,20 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 	workers := new(errgroup.Group)
 	workers.SetLimit(s.concurrency)
 
+	// if namespace is set and no images are specified, fetch all images in that namespace.
+	registryNamespace := s.conn.GetNamespace()
+	if registryNamespace != "" && len(s.conn.Images) == 0 {
+		start := time.Now()
+		namespaceImages, err := GetNamespaceImages(ctx, registryNamespace, s.conn.GetRegistryToken())
+		if err != nil {
+			return fmt.Errorf("failed to list namespace: %s images: %w", registryNamespace, err)
+		}
+
+		dockerListImagesAPIDuration.WithLabelValues(s.name).Observe(time.Since(start).Seconds())
+
+		s.conn.Images = append(s.conn.Images, namespaceImages...)
+	}
+
 	for _, image := range s.conn.GetImages() {
 		if common.IsDone(ctx) {
 			return nil
@@ -127,42 +141,42 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		imgInfo, err := s.processImage(ctx, image)
 		if err != nil {
 			ctx.Logger().Error(err, "error processing image", "image", image)
-			return nil
+			continue
 		}
 
-		ctx = context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
+		imageCtx := context.WithValues(ctx, "image", imgInfo.base, "tag", imgInfo.tag)
 
-		ctx.Logger().V(2).Info("scanning image history")
+		imageCtx.Logger().V(2).Info("scanning image history")
 
 		layers, err := imgInfo.image.Layers()
 		if err != nil {
-			ctx.Logger().Error(err, "error getting image layers")
-			return nil
+			imageCtx.Logger().Error(err, "error getting image layers")
+			continue
 		}
 
 		// Get history entries and associate them with layers
-		historyEntries, err := getHistoryEntries(ctx, imgInfo, layers)
+		historyEntries, err := getHistoryEntries(imageCtx, imgInfo, layers)
 		if err != nil {
-			ctx.Logger().Error(err, "error getting image history entries")
-			return nil
+			imageCtx.Logger().Error(err, "error getting image history entries")
+			continue
 		}
 
 		// Scan each history entry for secrets in build commands
 		for _, historyEntry := range historyEntries {
-			if err := s.processHistoryEntry(ctx, historyEntry, chunksChan); err != nil {
-				ctx.Logger().Error(err, "error processing history entry")
-				return nil
+			if err := s.processHistoryEntry(imageCtx, historyEntry, chunksChan); err != nil {
+				imageCtx.Logger().Error(err, "error processing history entry")
+				continue
 			}
 			dockerHistoryEntriesScanned.WithLabelValues(s.name).Inc()
 		}
 
-		ctx.Logger().V(2).Info("scanning image layers")
+		imageCtx.Logger().V(2).Info("scanning image layers")
 
 		// Process each layer concurrently
 		for _, layer := range layers {
 			workers.Go(func() error {
-				if err := s.processLayer(ctx, layer, imgInfo, chunksChan); err != nil {
-					ctx.Logger().Error(err, "error processing layer")
+				if err := s.processLayer(imageCtx, layer, imgInfo, chunksChan); err != nil {
+					imageCtx.Logger().Error(err, "error processing layer")
 					return nil
 				}
 				dockerLayersScanned.WithLabelValues(s.name).Inc()
@@ -172,8 +186,8 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 		}
 
 		if err := workers.Wait(); err != nil {
-			ctx.Logger().Error(err, "error processing layers")
-			return nil
+			imageCtx.Logger().Error(err, "error processing layers")
+			continue
 		}
 
 		dockerImagesScanned.WithLabelValues(s.name).Inc()
@@ -185,6 +199,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
 // processImage processes an individual image and prepares it for further processing.
 // It handles three image source types: remote registry, local daemon, and tarball file.
 func (s *Source) processImage(ctx context.Context, image string) (imageInfo, error) {
+	ctx.Logger().V(5).Info("Processing individual Image")
 	var (
 		imgInfo   imageInfo
 		imageName name.Reference
@@ -261,6 +276,7 @@ func (*Source) extractImageNameTagDigest(image string) (imageInfo, name.Referenc
 // getHistoryEntries collates an image's configuration history together with the
 // corresponding layer digests for any non-empty layers.
 func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer) ([]historyEntryInfo, error) {
+	ctx.Logger().V(5).Info("Getting history entries")
 	config, err := imgInfo.image.ConfigFile()
 	if err != nil {
 		return nil, err
@@ -307,6 +323,7 @@ func getHistoryEntries(ctx context.Context, imgInfo imageInfo, layers []v1.Layer
 // processHistoryEntry processes a history entry from the image configuration metadata.
 // It scans the CreatedBy field which contains the command used to create that layer.
 func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEntryInfo, chunksChan chan *sources.Chunk) error {
+	ctx.Logger().V(5).Info("Processing history entries")
 	// Create a descriptive identifier for this history entry
 	// There's no file name here, so we use a synthetic path
 	entryPath := fmt.Sprintf("image-metadata:history:%d:created-by", historyInfo.index)
@@ -337,6 +354,8 @@ func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEnt
 // processLayer processes an individual layer of an image.
 // It decompresses the layer and extracts all files for scanning.
 func (s *Source) processLayer(ctx context.Context, layer v1.Layer, imgInfo imageInfo, chunksChan chan *sources.Chunk) error {
+	ctx.Logger().V(5).Info("Processing layer")
+
 	layerInfo := layerInfo{
 		base: imgInfo.base,
 		tag:  imgInfo.tag,
@@ -508,6 +527,28 @@ func (s *Source) remoteOpts() ([]remote.Option, error) {
 	return opts, nil
 }
 
+func GetNamespaceImages(ctx context.Context, namespace, registryToken string) ([]string, error) {
+	ctx.Logger().V(5).Info("Getting namespace images")
+
+	registry := MakeRegistryFromNamespace(namespace)
+
+	// attach the registry authentication token, if one is available.
+	if registryToken != "" {
+		registry.WithRegistryToken(registryToken)
+	}
+
+	ctx.Logger().Info(fmt.Sprintf("using registry: %s", registry.Name()))
+
+	namespaceImages, err := registry.ListImages(ctx, namespace)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list namespace images: %w", err)
+	}
+
+	ctx.Logger().Info(fmt.Sprintf("namespace: %s has %d images", namespace, len(namespaceImages)))
+
+	return namespaceImages, nil
+}
+
 // baseAndTagFromImage extracts the base name and tag/digest from an image reference string.
 // It handles both digest-based references (image@sha256:...) and tag-based references (image:tag).
 func baseAndTagFromImage(image string) (base, tag string, hasDigest bool) {

pkg/sources/docker/README.md

@@ -31,4 +31,14 @@ var (
 		Help:      "Total number of Docker images scanned.",
 	},
 		[]string{"source_name"})
+
+	dockerListImagesAPIDuration = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: common.MetricsNamespace,
+			Subsystem: common.MetricsSubsystem,
+			Name:      "docker_list_images_api_duration_seconds",
+			Help:      "Duration of Docker list images API calls.",
+			Buckets:   prometheus.DefBuckets,
+		},
+		[]string{"source_name"})
 )