Merge branch 'main' into feat/rootlywebhook-detector

Author: shahzadhaider1

README.md

@@ -269,6 +269,8 @@ Run trufflehog from the parent directory (outside the git repo).
 $ trufflehog git file://test_keys --results=verified,unknown
 ```
 
+To guard against malicious git configs in local scanning (see CVE-2025-41390), TruffleHog clones local git repositories to a temporary directory prior to scanning. This follows [Git's security best practices](https://git-scm.com/docs/git#_security). If you want to specify a custom path to clone the repository to (instead of tmp), you can use the `--clone-path` flag. If you'd like to skip the local cloning process and scan the repository directly (only do this for trusted repos), you can use the `--trust-local-git-config` flag.
+
 ## 10: Scan GCS buckets for only verified secrets
 
 ```bash

go.mod

@@ -212,7 +212,6 @@ require (
 	github.com/golang-sql/sqlexp v0.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v1.0.0 // indirect
-	github.com/google/go-github/v69 v69.0.0 // indirect
 	github.com/google/go-github/v72 v72.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
@@ -253,8 +252,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/muesli/termenv v0.16.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
-	github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
-	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/nwaples/rardecode/v2 v2.2.1 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.1 // indirect

go.sum

@@ -91,20 +91,21 @@ var (
 	skipAdditionalRefs = cli.Flag("skip-additional-refs", "Skip additional references.").Bool()
 	userAgentSuffix    = cli.Flag("user-agent-suffix", "Suffix to add to User-Agent.").String()
 
-	gitScan             = cli.Command("git", "Find credentials in git repositories.")
-	gitScanURI          = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
-	gitScanIncludePaths = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
-	gitScanExcludePaths = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
-	gitScanExcludeGlobs = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
-	gitScanSinceCommit  = gitScan.Flag("since-commit", "Commit to start scan from.").String()
-	gitScanBranch       = gitScan.Flag("branch", "Branch to scan.").String()
-	gitScanMaxDepth     = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
-	gitScanBare         = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
-	gitClonePath        = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
-	gitNoCleanup        = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
-	_                   = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
-	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
-	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
+	gitScan                = cli.Command("git", "Find credentials in git repositories.")
+	gitScanURI             = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
+	gitScanIncludePaths    = gitScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
+	gitScanExcludePaths    = gitScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
+	gitScanExcludeGlobs    = gitScan.Flag("exclude-globs", "Comma separated list of globs to exclude in scan. This option filters at the `git log` level, resulting in faster scans.").String()
+	gitScanSinceCommit     = gitScan.Flag("since-commit", "Commit to start scan from.").String()
+	gitScanBranch          = gitScan.Flag("branch", "Branch to scan.").String()
+	gitScanMaxDepth        = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
+	gitScanBare            = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
+	gitClonePath           = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
+	gitNoCleanup           = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
+	gitTrustLocalGitConfig = gitScan.Flag("trust-local-git-config", "Trust local git config.").Bool()
+	_                      = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
+	_                      = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
+	_                      = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
 
 	githubScan                  = cli.Command("github", "Find credentials in GitHub repositories.")
 	githubScanEndpoint          = githubScan.Flag("endpoint", "GitHub endpoint.").Default("https://api.github.com").String()
@@ -735,35 +736,24 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 	var refs []sources.JobProgressRef
 	switch cmd {
 	case gitScan.FullCommand():
-		gitCloneTempPath = *gitClonePath
-		// validate the commit for local repository only
-		if *gitScanSinceCommit != "" && strings.HasPrefix(*gitScanURI, "file") {
-			if !isValidCommit(*gitScanURI, *gitScanSinceCommit) {
-				ctx.Logger().Info("Warning: The provided commit hash appears to be invalid.")
-			}
-		}
-
-		// clone path is only supported for HTTPS repository URLs, not for local repositories.
-		if *gitClonePath != "" && strings.HasPrefix(*gitScanURI, "file://") {
-			return scanMetrics, fmt.Errorf("invalid configuration: --clone-path cannot be used with a local repository URL")
-		}
 
 		if err := validateClonePath(*gitClonePath, *gitNoCleanup); err != nil {
 			return scanMetrics, err
 		}
 
 		gitCfg := sources.GitConfig{
-			URI:              *gitScanURI,
-			IncludePathsFile: *gitScanIncludePaths,
-			ExcludePathsFile: *gitScanExcludePaths,
-			HeadRef:          *gitScanBranch,
-			BaseRef:          *gitScanSinceCommit,
-			MaxDepth:         *gitScanMaxDepth,
-			Bare:             *gitScanBare,
-			ExcludeGlobs:     *gitScanExcludeGlobs,
-			ClonePath:        *gitClonePath,
-			NoCleanup:        *gitNoCleanup,
-			PrintLegacyJSON:  *jsonLegacy,
+			URI:                 *gitScanURI,
+			IncludePathsFile:    *gitScanIncludePaths,
+			ExcludePathsFile:    *gitScanExcludePaths,
+			HeadRef:             *gitScanBranch,
+			BaseRef:             *gitScanSinceCommit,
+			MaxDepth:            *gitScanMaxDepth,
+			Bare:                *gitScanBare,
+			ExcludeGlobs:        *gitScanExcludeGlobs,
+			ClonePath:           *gitClonePath,
+			NoCleanup:           *gitNoCleanup,
+			PrintLegacyJSON:     *jsonLegacy,
+			TrustLocalGitConfig: *gitTrustLocalGitConfig,
 		}
 		if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
@@ -1168,18 +1158,6 @@ func printAverageDetectorTime(e *engine.Engine) {
 	}
 }
 
-// Function to check if the commit is valid
-func isValidCommit(uri, commit string) bool {
-	// handle file:// urls
-	repoPath, _ := strings.CutPrefix(uri, "file://") // remove the prefix to validate against the repo path
-	output, err := exec.Command("git", "-C", repoPath, "cat-file", "-t", commit).Output()
-	if err != nil {
-		return false
-	}
-
-	return strings.TrimSpace(string(output)) == "commit"
-}
-
 // validateClonePath ensures that --clone-path, if provided, exists and is a directory.
 // It also verifies that --no-cleanup is only allowed when --clone-path is set.
 // Note: without a custom clone path, repositories are cloned into temporary directories, which should never be retained.

main.go

@@ -28,11 +28,14 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
 var (
 	// Oauth token
 	// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
-	// the middle regex `(?:[a-zA-Z0-9.\/?=&:-]{0,40})` is to match the prefix of token match to avoid processing common known patterns
-	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b(?:[a-zA-Z0-9.\/?=&:-]{0,40})([a-f0-9]{40})\b`)
-
-	// TODO: Oauth2 client_id and client_secret
-	// https://developer.github.com/v3/#oauth2-keysecret
+	keyPat = regexp.MustCompile(
+		detectors.PrefixRegex([]string{
+			"github_token", "github_secret", "github_key", "github_api", "github_pat",
+			"githubtoken", "githubsecret", "githubkey", "githubapi", "githubpat",
+			"gh_token", "gh_secret", "gh_key", "gh_api", "gh_pat",
+			"ghtoken", "ghsecret", "ghkey", "ghapi", "ghpat",
+		}) + `\b([0-9a-f]{40})\b`,
+	)
 )
 
 // TODO: Add secret context?? Information about access, ownership etc

pkg/detectors/github/v1/github_old.go

@@ -15,18 +15,19 @@ import (
 // ScanGit scans any git source.
 func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) (sources.JobProgressRef, error) {
 	connection := &sourcespb.Git{
-		Head:             c.HeadRef,
-		Base:             c.BaseRef,
-		Bare:             c.Bare,
-		Uri:              c.URI,
-		ExcludeGlobs:     c.ExcludeGlobs,
-		IncludePathsFile: c.IncludePathsFile,
-		ExcludePathsFile: c.ExcludePathsFile,
-		MaxDepth:         int64(c.MaxDepth),
-		SkipBinaries:     c.SkipBinaries,
-		ClonePath:        c.ClonePath,
-		NoCleanup:        c.NoCleanup,
-		PrintLegacyJson:  c.PrintLegacyJSON,
+		Head:                c.HeadRef,
+		Base:                c.BaseRef,
+		Bare:                c.Bare,
+		Uri:                 c.URI,
+		ExcludeGlobs:        c.ExcludeGlobs,
+		IncludePathsFile:    c.IncludePathsFile,
+		ExcludePathsFile:    c.ExcludePathsFile,
+		MaxDepth:            int64(c.MaxDepth),
+		SkipBinaries:        c.SkipBinaries,
+		ClonePath:           c.ClonePath,
+		NoCleanup:           c.NoCleanup,
+		PrintLegacyJson:     c.PrintLegacyJSON,
+		TrustLocalGitConfig: c.TrustLocalGitConfig,
 	}
 
 	var conn anypb.Any

pkg/engine/git.go

@@ -1,16 +1,21 @@
 package engine
 
 import (
+	"net/url"
 	"os"
+	"os/exec"
+	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine/defaults"
+	"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
@@ -32,7 +37,7 @@ func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) e
 func TestGitEngine(t *testing.T) {
 	ctx := context.Background()
 	repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
-	path, _, err := git.PrepareRepo(ctx, repoUrl, "")
+	path, _, err := git.PrepareRepo(ctx, repoUrl, "", false, false)
 	if err != nil {
 		t.Error(err)
 	}
@@ -118,6 +123,78 @@ func TestGitEngine(t *testing.T) {
 	}
 }
 
+func TestGitEngineWithMirrorAndBareClones(t *testing.T) {
+	ctx := context.Background()
+
+	parent, err := os.MkdirTemp("", "trufflehog-test-keys-*")
+	if err != nil {
+		t.Fail()
+	}
+	defer os.RemoveAll(parent)
+	localRepo := filepath.Join(parent, "test_keys.git")
+	cloneCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+
+	// clone with --mirror and --bare from https://github.com/trufflesecurity/test_keys.git to local and then pass it in as a local path
+	cloneCmd := exec.CommandContext(cloneCtx, "git", "clone", "--mirror", "--bare", "https://github.com/trufflesecurity/test_keys.git", localRepo)
+	if _, err := cloneCmd.CombinedOutput(); err != nil {
+		t.Fail()
+	}
+
+	fileURI := (&url.URL{Scheme: "file", Path: filepath.ToSlash(localRepo)}).String()
+
+	run := func(t *testing.T, mirror bool, cfg sources.GitConfig) (uint64, uint64) {
+		t.Helper()
+
+		const defaultOutputBufferSize = 64
+		opts := []func(*sources.SourceManager){
+			sources.WithSourceUnits(),
+			sources.WithBufferedOutput(defaultOutputBufferSize),
+		}
+		sourceManager := sources.NewManager(opts...)
+
+		conf := Config{
+			Concurrency:   1,
+			Decoders:      decoders.DefaultDecoders(),
+			Detectors:     defaults.DefaultDetectors(),
+			Verify:        false, // avoid network-dependent verification in tests
+			SourceManager: sourceManager,
+			Dispatcher:    NewPrinterDispatcher(new(discardPrinter)),
+		}
+
+		feature.UseGitMirror.Store(false)
+		if mirror {
+			feature.UseGitMirror.Store(true)
+			defer feature.UseGitMirror.Store(false)
+		}
+
+		e, err := NewEngine(ctx, &conf)
+		assert.NoError(t, err)
+
+		e.Start(ctx)
+		_, err = e.ScanGit(ctx, cfg)
+		assert.NoError(t, err)
+		assert.NoError(t, e.Finish(ctx))
+
+		m := e.GetMetrics()
+		secrets := m.VerifiedSecretsFound + m.UnverifiedSecretsFound
+		bytes := m.BytesScanned
+		return secrets, bytes
+	}
+
+	s1, b1 := run(t, true, sources.GitConfig{URI: "https://github.com/trufflesecurity/test_keys.git"})
+	s2, b2 := run(t, false, sources.GitConfig{URI: fileURI, Bare: true})
+	s3, b3 := run(t, false, sources.GitConfig{URI: fileURI, Bare: true, TrustLocalGitConfig: true})
+
+	assert.Greater(t, int(s1), 0)
+	assert.Greater(t, int(b1), 0)
+
+	assert.Equal(t, s1, s2)
+	assert.Equal(t, s1, s3)
+	assert.Equal(t, b1, b2)
+	assert.Equal(t, b1, b3)
+}
+
 func BenchmarkGitEngine(b *testing.B) {
 	ctx := context.Background()
 	repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"