Skip to content

Commit 7aa4c79

Browse files
G-RathG-Rath
authored
feat: support providing parse-as per lockfile / directory (#189)
1 parent a9867c9 commit 7aa4c79

File tree

3 files changed

+220
-16
lines changed

3 files changed

+220
-16
lines changed

‎README.md‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,23 @@ that is not a dependency specification (e.g. flags or files in the case of
7777
`requirements.txt`, though `<properties>` _is_ supported for `pom.xml`)
7878

7979
The detector will attempt to automatically determine the parser to use for each
80-
file based on the filename - you can manually specify the parser to use for all
81-
files with the `-parse-as` flag:
80+
file based on the filename - you can also explicitly specify the parser to use
81+
by prefixing it to the start of the given path, seperated with an `:` symbol:
82+
83+
```shell
84+
osv-detector requirements.txt:path/to/my/requirements/ requirements.txt:path/to/my/file.txt
85+
```
86+
87+
If you have a path with a colon in its name, you can with just a colon to
88+
explicitly signal to the detector that it should infer the parser based on the
89+
filename:
90+
91+
```shell
92+
osv-detector ':/path/to/my:projects/package-lock.json'
93+
```
94+
95+
You can also set the default parser to use for all files with the `--parse-as`
96+
flag:
8297

8398
```shell
8499
osv-detector --parse-as 'package-lock.json' path/to/my/file.lock

‎main.go‎

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"os"
1414
"path/filepath"
1515
"sort"
16+
"strings"
1617
)
1718

1819
// these come from goreleaser
@@ -283,31 +284,39 @@ func findLockfiles(r *reporter.Reporter, pathToLockOrDirectory string, parseAs s
283284
return lockfiles, err != nil
284285
}
285286

286-
func findAllLockfiles(r *reporter.Reporter, pathsToCheck []string, parseAs string) ([]string, bool) {
287+
func findAllLockfiles(r *reporter.Reporter, pathsToCheck []string, parseAsGlobal string) ([]string, bool) {
287288
var paths []string
288289

289-
if parseAs == parseAsCsvRow {
290-
return []string{"-"}, false
290+
if parseAsGlobal == parseAsCsvRow {
291+
return []string{parseAsCsvRow + ":-"}, false
291292
}
292293

293294
errored := false
294295

295296
for _, pathToLockOrDirectory := range pathsToCheck {
297+
parseAs, pathToLockOrDirectory := parseLockfilePathWithParseAs(pathToLockOrDirectory)
298+
299+
if parseAs == "" {
300+
parseAs = parseAsGlobal
301+
}
302+
296303
lps, erred := findLockfiles(r, pathToLockOrDirectory, parseAs)
297304

298305
if erred {
299306
errored = true
300307
}
301308

302309
for _, p := range lps {
303-
paths = append(paths, filepath.Clean(p))
310+
paths = append(paths, parseAs+":"+filepath.Clean(p))
304311
}
305312
}
306313

307314
return paths, errored
308315
}
309316

310-
func parseLockfile(pathToLock string, parseAs string, args []string) (lockfile.Lockfile, error) {
317+
func parseLockfile(pathToLock string, args []string) (lockfile.Lockfile, error) {
318+
parseAs, pathToLock := parseLockfilePathWithParseAs(pathToLock)
319+
311320
if parseAs == parseAsCsvRow {
312321
l, err := lockfile.FromCSVRows(pathToLock, parseAs, args)
313322

@@ -405,17 +414,28 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
405414
}
406415
}
407416

417+
func parseLockfilePathWithParseAs(lockfilePathWithParseAs string) (string, string) {
418+
if !strings.Contains(lockfilePathWithParseAs, ":") {
419+
lockfilePathWithParseAs = ":" + lockfilePathWithParseAs
420+
}
421+
422+
splits := strings.SplitN(lockfilePathWithParseAs, ":", 2)
423+
424+
return splits[0], splits[1]
425+
}
426+
408427
func readAllLockfiles(
409428
r *reporter.Reporter,
410-
pathsToLocks []string,
411-
parseAs string,
429+
pathsToLocksWithParseAs []string,
412430
args []string,
413431
checkForLocalConfig bool,
414432
config *configer.Config,
415433
) lockfileAndConfigOrErrs {
416-
lockfiles := make([]lockfileAndConfigOrErr, 0, len(pathsToLocks))
434+
lockfiles := make([]lockfileAndConfigOrErr, 0, len(pathsToLocksWithParseAs))
435+
436+
for _, pathToLockWithParseAs := range pathsToLocksWithParseAs {
437+
_, pathToLock := parseLockfilePathWithParseAs(pathToLockWithParseAs)
417438

418-
for _, pathToLock := range pathsToLocks {
419439
if checkForLocalConfig {
420440
base := filepath.Dir(pathToLock)
421441
con, err := configer.Find(r, base)
@@ -442,7 +462,7 @@ func readAllLockfiles(
442462
}
443463
}
444464

445-
lockf, err := parseLockfile(pathToLock, parseAs, args)
465+
lockf, err := parseLockfile(pathToLockWithParseAs, args)
446466
lockfiles = append(lockfiles, lockfileAndConfigOrErr{lockf, config, err})
447467
}
448468

@@ -522,6 +542,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
522542
return 0
523543
}
524544

545+
// ensure that if the global parseAs is set, it is one of the supported values
525546
if *parseAs != "" && *parseAs != parseAsCsvFile && *parseAs != parseAsCsvRow {
526547
if parser, parsedAs := lockfile.FindParser("", *parseAs); parser == nil {
527548
r.PrintError(fmt.Sprintf("Don't know how to parse files as \"%s\" - supported values are:\n", parsedAs))
@@ -537,9 +558,9 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
537558
}
538559
}
539560

540-
pathsToLocks, errored := findAllLockfiles(r, cli.Args(), *parseAs)
561+
pathsToLocksWithParseAs, errored := findAllLockfiles(r, cli.Args(), *parseAs)
541562

542-
if len(pathsToLocks) == 0 {
563+
if len(pathsToLocksWithParseAs) == 0 {
543564
r.PrintError(
544565
"You must provide at least one path to either a lockfile or a directory containing at least one lockfile (see --help for usage and flags)\n",
545566
)
@@ -576,7 +597,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
576597
loadLocalConfig = false
577598
}
578599

579-
files := readAllLockfiles(r, pathsToLocks, *parseAs, cli.Args(), loadLocalConfig, &config)
600+
files := readAllLockfiles(r, pathsToLocksWithParseAs, cli.Args(), loadLocalConfig, &config)
580601

581602
files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases)
582603

‎main_test.go‎

Lines changed: 169 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -499,7 +499,140 @@ func TestRun_DBs(t *testing.T) {
499499
}
500500
}
501501

502-
func TestRun_ParseAs(t *testing.T) {
502+
func TestRun_ParseAsSpecific(t *testing.T) {
503+
t.Parallel()
504+
505+
tests := []cliTestCase{
506+
// when there is just a ":", it defaults as empty
507+
{
508+
name: "",
509+
args: []string{filepath.FromSlash(":./fixtures/locks-insecure/composer.lock")},
510+
wantExitCode: 0,
511+
wantStdout: `
512+
Loaded the following OSV databases:
513+
514+
fixtures/locks-insecure/composer.lock: found 0 packages
515+
516+
no known vulnerabilities found
517+
`,
518+
wantStderr: "",
519+
},
520+
// ":" can be used as an escape (no test though because it's invalid on Windows)
521+
{
522+
name: "",
523+
args: []string{filepath.FromSlash(":./fixtures/locks-insecure/my:file")},
524+
wantExitCode: 127,
525+
wantStdout: "",
526+
wantStderr: `
527+
Error reading ./fixtures/locks-insecure/my:file: open ./fixtures/locks-insecure/my:file: %%
528+
You must provide at least one path to either a lockfile or a directory containing at least one lockfile (see --help for usage and flags)
529+
`,
530+
},
531+
// when a path to a file is given, parse-as is applied to that file
532+
{
533+
name: "",
534+
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-insecure/my-package-lock.json")},
535+
wantExitCode: 1,
536+
wantStdout: `
537+
Loaded the following OSV databases:
538+
npm (%% vulnerabilities, including withdrawn - last updated %%)
539+
540+
fixtures/locks-insecure/my-package-lock.json: found 1 package
541+
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
542+
543+
[email protected] is affected by the following vulnerabilities:
544+
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
545+
546+
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
547+
`,
548+
wantStderr: "",
549+
},
550+
// when a path to a directory is given, parse-as is applied to all files in the directory
551+
{
552+
name: "",
553+
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-insecure")},
554+
wantExitCode: 1,
555+
wantStdout: `
556+
Loaded the following OSV databases:
557+
npm (%% vulnerabilities, including withdrawn - last updated %%)
558+
559+
fixtures/locks-insecure/composer.lock: found 0 packages
560+
561+
no known vulnerabilities found
562+
563+
fixtures/locks-insecure/my-package-lock.json: found 1 package
564+
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
565+
566+
[email protected] is affected by the following vulnerabilities:
567+
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
568+
569+
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
570+
`,
571+
wantStderr: "",
572+
},
573+
// files that error on parsing don't stop parsable files from being checked
574+
{
575+
name: "",
576+
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-empty")},
577+
wantExitCode: 127,
578+
wantStdout: `
579+
Loaded the following OSV databases:
580+
581+
582+
fixtures/locks-empty/composer.lock: found 0 packages
583+
584+
no known vulnerabilities found
585+
586+
`,
587+
wantStderr: `
588+
Error, could not parse fixtures/locks-empty/Gemfile.lock: unexpected end of JSON input
589+
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
590+
`,
591+
},
592+
// files that error on parsing don't stop parsable files from being checked
593+
{
594+
name: "",
595+
args: []string{filepath.FromSlash("package-lock.json:./fixtures/locks-empty"), filepath.FromSlash("package-lock.json:./fixtures/locks-insecure")},
596+
wantExitCode: 127,
597+
wantStdout: `
598+
Loaded the following OSV databases:
599+
npm (%% vulnerabilities, including withdrawn - last updated %%)
600+
601+
602+
fixtures/locks-empty/composer.lock: found 0 packages
603+
604+
no known vulnerabilities found
605+
606+
607+
fixtures/locks-insecure/composer.lock: found 0 packages
608+
609+
no known vulnerabilities found
610+
611+
fixtures/locks-insecure/my-package-lock.json: found 1 package
612+
Using db npm (%% vulnerabilities, including withdrawn - last updated %%)
613+
614+
[email protected] is affected by the following vulnerabilities:
615+
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
616+
617+
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
618+
`,
619+
wantStderr: `
620+
Error, could not parse fixtures/locks-empty/Gemfile.lock: unexpected end of JSON input
621+
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
622+
`,
623+
},
624+
}
625+
for _, tt := range tests {
626+
tt := tt
627+
t.Run(tt.name, func(t *testing.T) {
628+
t.Parallel()
629+
630+
testCli(t, tt)
631+
})
632+
}
633+
}
634+
635+
func TestRun_ParseAsGlobal(t *testing.T) {
503636
t.Parallel()
504637

505638
tests := []cliTestCase{
@@ -596,6 +729,41 @@ func TestRun_ParseAs(t *testing.T) {
596729
Error, could not parse fixtures/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
597730
`,
598731
},
732+
// specific parse-as takes precedence over global parse-as
733+
{
734+
name: "",
735+
args: []string{"--parse-as", "package-lock.json", filepath.FromSlash("Gemfile.lock:./fixtures/locks-empty"), filepath.FromSlash("./fixtures/locks-insecure")},
736+
wantExitCode: 1,
737+
wantStdout: `
738+
Loaded the following OSV databases:
739+
npm (2971 vulnerabilities, including withdrawn - last updated %%)
740+
741+
fixtures/locks-empty/Gemfile.lock: found 0 packages
742+
743+
no known vulnerabilities found
744+
745+
fixtures/locks-empty/composer.lock: found 0 packages
746+
747+
no known vulnerabilities found
748+
749+
fixtures/locks-empty/yarn.lock: found 0 packages
750+
751+
no known vulnerabilities found
752+
753+
fixtures/locks-insecure/composer.lock: found 0 packages
754+
755+
no known vulnerabilities found
756+
757+
fixtures/locks-insecure/my-package-lock.json: found 1 package
758+
Using db npm (2971 vulnerabilities, including withdrawn - last updated %%)
759+
760+
[email protected] is affected by the following vulnerabilities:
761+
GHSA-whgm-jr23-g3j9: Uncontrolled Resource Consumption in ansi-html (https://github.com/advisories/GHSA-whgm-jr23-g3j9)
762+
763+
1 known vulnerability found in fixtures/locks-insecure/my-package-lock.json
764+
`,
765+
wantStderr: "",
766+
},
599767
}
600768
for _, tt := range tests {
601769
tt := tt

0 commit comments

Comments
 (0)