feat: support -r flag in requirements.txt files (#174)

Author: G-Rath

pkg/lockfile/fixtures/pip/file-format-example.txt

@@ -12,7 +12,7 @@ coverage != 3.5             # Version Exclusion. Anything except version 3.5
 Mopidy-Dirble ~= 1.1        # Compatible release. Same as >= 1.1, == 1.*
 
 ###### Refer to other requirements files ######
--r other-requirements.txt
+-r other-file.txt
 
 ###### A particular file ######
 ./downloads/numpy-1.9.2-cp34-none-win32.whl

pkg/lockfile/fixtures/pip/other-file.txt

@@ -0,0 +1 @@
+django==2.2.24

pkg/lockfile/fixtures/pip/with-bad-r-option.txt

@@ -0,0 +1,3 @@
+requests==1.2.3
+
+-r ./does-not-exist.txt

pkg/lockfile/fixtures/pip/with-multiple-r-options.txt

@@ -0,0 +1,5 @@
+-r ./one-package-constrained.txt
+-r ./multiple-packages-mixed.txt
+
+requests==1.2.3
+pandas==0.23.4

pkg/lockfile/parse-requirements-txt.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 )
@@ -91,11 +92,11 @@ func isNotRequirementLine(line string) bool {
 }
 
 func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) {
-	var packages []PackageDetails
+	packages := map[string]PackageDetails{}
 
 	file, err := os.Open(pathToLockfile)
 	if err != nil {
-		return packages, fmt.Errorf("could not open %s: %w", pathToLockfile, err)
+		return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err)
 	}
 	defer file.Close()
 
@@ -104,16 +105,33 @@ func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) {
 	for scanner.Scan() {
 		line := removeComments(scanner.Text())
 
+		if strings.HasPrefix(line, "-r ") {
+			details, err := ParseRequirementsTxt(
+				filepath.Join(filepath.Dir(pathToLockfile), strings.TrimPrefix(line, "-r ")),
+			)
+
+			if err != nil {
+				return []PackageDetails{}, fmt.Errorf("failed to include %s: %w", line, err)
+			}
+
+			for _, detail := range details {
+				packages[detail.Name+"@"+detail.Version] = detail
+			}
+
+			continue
+		}
+
 		if isNotRequirementLine(line) {
 			continue
 		}
 
-		packages = append(packages, parseLine(line))
+		detail := parseLine(line)
+		packages[detail.Name+"@"+detail.Version] = detail
 	}
 
 	if err := scanner.Err(); err != nil {
-		return packages, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err)
+		return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err)
 	}
 
-	return packages, nil
+	return pkgDetailsMapToSlice(packages), nil
 }

pkg/lockfile/parse-requirements-txt_test.go

@@ -292,6 +292,12 @@ func TestParseRequirementsTxt_FileFormatExample(t *testing.T) {
 			Ecosystem: lockfile.PipEcosystem,
 			CompareAs: lockfile.PipEcosystem,
 		},
+		{
+			Name:      "django",
+			Version:   "2.2.24",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
 	})
 }
 
@@ -344,3 +350,85 @@ func TestParseRequirementsTxt_NonNormalizedNames(t *testing.T) {
 		},
 	})
 }
+
+func TestParseRequirementsTxt_WithMultipleROptions(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-multiple-r-options.txt")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "flask",
+			Version:   "0.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "flask-cors",
+			Version:   "0.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "pandas",
+			Version:   "0.23.4",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "numpy",
+			Version:   "1.16.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "scikit-learn",
+			Version:   "0.20.1",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "sklearn",
+			Version:   "0.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "requests",
+			Version:   "0.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "gevent",
+			Version:   "0.0.0",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "requests",
+			Version:   "1.2.3",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "django",
+			Version:   "2.2.24",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+	})
+}
+
+func TestParseRequirementsTxt_WithBadROption(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/with-bad-r-option.txt")
+
+	expectErrContaining(t, err, "could not open")
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}