fix: handle line continuations in requirements.txt files (#195)

Author: G-Rath

pkg/lockfile/fixtures/pip/line-continuation.txt

@@ -0,0 +1,17 @@
+# unescaped
+foo==\
+\
+ \
+  \
+1.2.3
+
+# escaped, a literal backslash for some reason
+bar == 4.5\\
+.6
+
+# comments are stripped only after line continuations are processed
+baz == 7.8.9 # \
+baz == 1.2.3
+
+# continue to end
+qux == 10.11.12\

pkg/lockfile/parse-requirements-txt.go

@@ -91,6 +91,14 @@ func isNotRequirementLine(line string) bool {
 		strings.HasPrefix(line, "/")
 }
 
+func isLineContinuation(line string) bool {
+	// checks that the line ends with an odd number of backslashes,
+	// meaning the last one isn't escaped
+	var re = regexp.MustCompile(`([^\\]|^)(\\{2})*\\$`)
+
+	return re.MatchString(line)
+}
+
 func ParseRequirementsTxt(pathToLockfile string) ([]PackageDetails, error) {
 	return parseRequirementsTxt(pathToLockfile, map[string]struct{}{})
 }
@@ -107,7 +115,17 @@ func parseRequirementsTxt(pathToLockfile string, requiredAlready map[string]stru
 	scanner := bufio.NewScanner(file)
 
 	for scanner.Scan() {
-		line := removeComments(scanner.Text())
+		line := scanner.Text()
+
+		for isLineContinuation(line) {
+			line = strings.TrimSuffix(line, "\\")
+
+			if scanner.Scan() {
+				line += scanner.Text()
+			}
+		}
+
+		line = removeComments(line)
 
 		if ar := strings.TrimPrefix(line, "-r "); ar != line {
 			ar = filepath.Join(filepath.Dir(pathToLockfile), ar)

pkg/lockfile/parse-requirements-txt_test.go

@@ -525,3 +525,40 @@ func TestParseRequirementsTxt_CyclicRComplex(t *testing.T) {
 		},
 	})
 }
+
+func TestParseRequirementsTxt_LineContinuation(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseRequirementsTxt("fixtures/pip/line-continuation.txt")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "foo",
+			Version:   "1.2.3",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "bar",
+			Version:   "4.5\\\\",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "baz",
+			Version:   "7.8.9",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+		{
+			Name:      "qux",
+			Version:   "10.11.12",
+			Ecosystem: lockfile.PipEcosystem,
+			CompareAs: lockfile.PipEcosystem,
+		},
+	})
+}