From de1fa0051d8e40c8ce22aa47147b727ecd9241d2 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 14 Aug 2025 11:21:28 +0000 Subject: [PATCH 01/16] implement URL normalization function and enhance SSRF detection --- .../context/event_getters.go | 2 ++ .../helpers/normalizeRequestUrl.go | 28 +++++++++++++++++++ .../helpers/normalizeRequestUrl_test.go | 21 ++++++++++++++ lib/request-processor/helpers/tryParseURL.go | 14 +++++++++- .../ssrf/findHostnameInUserInput.go | 21 ++++++++++++-- .../ssrf/findHostnameInUserInput_test.go | 14 ++++++++++ 6 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 lib/request-processor/helpers/normalizeRequestUrl.go create mode 100644 lib/request-processor/helpers/normalizeRequestUrl_test.go diff --git a/lib/request-processor/context/event_getters.go b/lib/request-processor/context/event_getters.go index 513479fdf..07c885e71 100644 --- a/lib/request-processor/context/event_getters.go +++ b/lib/request-processor/context/event_getters.go @@ -49,6 +49,8 @@ func GetModule() string { func getHostNameAndPort(urlCallbackId int) (string, uint32) { // urlcallbackid is the type of data we request, eg C.OUTGOING_REQUEST_URL urlStr := Context.Callback(urlCallbackId) + // remove all control characters (< 32) and 0x7f(DEL) also replace \@ with @ and remove all whitespace + urlStr = helpers.NormalizeRawUrl(urlStr) urlParsed, err := url.Parse(urlStr) if err != nil { return "", 0 diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go new file mode 100644 index 000000000..2412275e3 --- /dev/null +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -0,0 +1,28 @@ +package helpers + +import "strings" + +func removeCTLByte(urlStr string) string { + for i := 0; i < len(urlStr); i++ { + if urlStr[i] < ' ' || urlStr[i] == 0x7f { + urlStr = urlStr[:i] + urlStr[i+1:] + } + } + return urlStr +} + +// \@ -> @ +func removeBackslashAt(urlStr string) string { + return strings.ReplaceAll(urlStr, "\\@", "@") +} + +func removeWhitespace(urlStr string) string { + return strings.ReplaceAll(urlStr, " ", "") +} + +func NormalizeRawUrl(urlStr string) string { + urlStr = removeCTLByte(urlStr) + urlStr = removeBackslashAt(urlStr) + urlStr = removeWhitespace(urlStr) + return urlStr +} diff --git a/lib/request-processor/helpers/normalizeRequestUrl_test.go b/lib/request-processor/helpers/normalizeRequestUrl_test.go new file mode 100644 index 000000000..4a5610397 --- /dev/null +++ b/lib/request-processor/helpers/normalizeRequestUrl_test.go @@ -0,0 +1,21 @@ +package helpers + +import "testing" + +func TestNormalizeRequestUrl(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"http://localhost:4000", "http://localhost:4000"}, + {"http://localhost:4000 ", "http://localhost:4000"}, + {"http://localhost:4000" + "\x00", "http://localhost:4000"}, + {"http://\\@localhost:4000", "http://@localhost:4000"}, + } + for _, test := range tests { + result := NormalizeRawUrl(test.input) + if result != test.expected { + t.Errorf("For input '%s', expected %v but got %v", test.input, test.expected, result) + } + } +} diff --git a/lib/request-processor/helpers/tryParseURL.go b/lib/request-processor/helpers/tryParseURL.go index 648b59074..2e903bfe6 100644 --- a/lib/request-processor/helpers/tryParseURL.go +++ b/lib/request-processor/helpers/tryParseURL.go @@ -1,13 +1,14 @@ package helpers import ( + "net" "net/url" "golang.org/x/net/idna" ) func TryParseURL(input string) *url.URL { - parsedURL, err := url.ParseRequestURI(input) + parsedURL, err := url.Parse(input) if err != nil { return nil } @@ -17,5 +18,16 @@ func TryParseURL(input string) *url.URL { if err == nil { parsedURL.Host = parsedHost } + + host, port, err := net.SplitHostPort(parsedURL.Host) + if err == nil { + ip := net.ParseIP(host) + if ip != nil { + parsedURL.Host = ip.String() + ":" + port + } else { + parsedURL.Host = host + ":" + port + } + } + return parsedURL } diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go index 24eb55307..a970d16ac 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go @@ -2,25 +2,42 @@ package ssrf import ( "main/helpers" + "main/log" + "net/url" + "strconv" "strings" ) func findHostnameInUserInput(userInput string, hostname string, port uint32) bool { - + userInput = helpers.NormalizeRawUrl(userInput) + log.Debugf("findHostnameInUserInput: userInput: %s, hostname: %s, port: %d", userInput, hostname, port) if len(userInput) <= 1 { return false } + // if hostname contains : we need to add the [ and ] to the hostname (ipv6) + if strings.Contains(hostname, ":") { + hostname = "[" + hostname + "]" + } - hostnameURL := helpers.TryParseURL("http://" + hostname) + hostnameURL := helpers.TryParseURL("http://" + hostname + ":" + strconv.Itoa(int(port))) if hostnameURL == nil { return false } userInput = helpers.ExtractResourceOrOriginal(userInput) variants := []string{userInput, "http://" + userInput, "https://" + userInput} + // if decoded user input is different, we need to add the decoded variant to the variants + decodedUserInput, err := url.QueryUnescape(userInput) + if err == nil && decodedUserInput != userInput { + variants = append(variants, decodedUserInput, "http://"+decodedUserInput, "https://"+decodedUserInput) + } for _, variant := range variants { userInputURL := helpers.TryParseURL(variant) + if userInputURL == nil { + continue + } + // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 // "The host subcomponent is case-insensitive." if userInputURL != nil && strings.EqualFold(userInputURL.Hostname(), hostnameURL.Hostname()) { diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index 17e75ee0c..8a7b339f8 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,6 +11,20 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ + {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "0000:0000:0000:0000:0000:0000:0000:0001", 4000, true}, + {"http://127.0.0.1:8080#\\@127.2.2.2:80/ ", "127.0.0.1", 8080, true}, + {"http://1.1.1.1 &@127.0.0.1:4000# @3.3.3.3/", "127.0.0.1", 4000, true}, + {"http://127.1.1.1:4000:\\@@127.0.0.1:8080/", "127.0.0.1", 8080, true}, + {"http://127.1.1.1:4000\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, + {"http://%31%32%37.%30.%30.%31:4000", "127.0.0.1", 4000, true}, + {"http://%30:4000", "0", 4000, true}, + {"http://127%2E0%2E0%2E1:4000", "127.0.0.1", 4000, true}, + {"http://[::ffff:127.0.0.1]:4000", "::ffff:127.0.0.1", 4000, true}, + {"http://[0:0:0:0:0:0:0:1]:4000", "::1", 4000, true}, + {"http://[::]:4000", "::", 4000, true}, + {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "::1", 4000, true}, + {"http://[::1]:4000", "::1", 4000, true}, + {"http://[0:0::1]:4000", "::1", 4000, true}, {"https://m%C3%BCnchen.de", "münchen.de", 0, true}, {"https://münchen.de", "xn--mnchen-3ya.de", 0, true}, {"https://xn--mnchen-3ya.de", "münchen.de", 0, true}, From d42f4973c4424d456af14fcf7c069fd0a0f29f46 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 14 Aug 2025 11:54:32 +0000 Subject: [PATCH 02/16] add error checking in URL parsing --- lib/request-processor/helpers/normalizeRequestUrl.go | 3 ++- lib/request-processor/helpers/tryParseURL.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index 2412275e3..f10d8b30f 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -11,7 +11,8 @@ func removeCTLByte(urlStr string) string { return urlStr } -// \@ -> @ +// If the urlStr contains \@ we need to replace it with @ +// because the URL.Parse will fail to parse the url (invalid userinfo) func removeBackslashAt(urlStr string) string { return strings.ReplaceAll(urlStr, "\\@", "@") } diff --git a/lib/request-processor/helpers/tryParseURL.go b/lib/request-processor/helpers/tryParseURL.go index 2e903bfe6..a6660252c 100644 --- a/lib/request-processor/helpers/tryParseURL.go +++ b/lib/request-processor/helpers/tryParseURL.go @@ -9,7 +9,7 @@ import ( func TryParseURL(input string) *url.URL { parsedURL, err := url.Parse(input) - if err != nil { + if err != nil || parsedURL.Host == "" { return nil } From 7bb57a4dc5551b7b0a2d0b45821cc8a1e8f3d579 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 14 Aug 2025 12:03:57 +0000 Subject: [PATCH 03/16] refactor --- .../ssrf/findHostnameInUserInput.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go index a970d16ac..b12e2f65d 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go @@ -8,6 +8,15 @@ import ( "strings" ) +func getVariants(str string) []string { + variants := []string{str, "http://" + str, "https://" + str} + decodedUserInput, err := url.QueryUnescape(str) + if err == nil && decodedUserInput != str { + variants = append(variants, decodedUserInput, "http://"+decodedUserInput, "https://"+decodedUserInput) + } + return variants +} + func findHostnameInUserInput(userInput string, hostname string, port uint32) bool { userInput = helpers.NormalizeRawUrl(userInput) log.Debugf("findHostnameInUserInput: userInput: %s, hostname: %s, port: %d", userInput, hostname, port) @@ -25,12 +34,7 @@ func findHostnameInUserInput(userInput string, hostname string, port uint32) boo } userInput = helpers.ExtractResourceOrOriginal(userInput) - variants := []string{userInput, "http://" + userInput, "https://" + userInput} - // if decoded user input is different, we need to add the decoded variant to the variants - decodedUserInput, err := url.QueryUnescape(userInput) - if err == nil && decodedUserInput != userInput { - variants = append(variants, decodedUserInput, "http://"+decodedUserInput, "https://"+decodedUserInput) - } + variants := getVariants(userInput) for _, variant := range variants { userInputURL := helpers.TryParseURL(variant) From 6d5e7f4a73032ca1e2fe8b51899aa767b1d35ed4 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 14 Aug 2025 12:47:34 +0000 Subject: [PATCH 04/16] add default ports for HTTP and HTTPS schemes --- lib/request-processor/helpers/tryParseURL.go | 13 ++++++++++++- .../ssrf/findHostnameInUserInput_test.go | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/request-processor/helpers/tryParseURL.go b/lib/request-processor/helpers/tryParseURL.go index a6660252c..b1323de0a 100644 --- a/lib/request-processor/helpers/tryParseURL.go +++ b/lib/request-processor/helpers/tryParseURL.go @@ -3,6 +3,7 @@ package helpers import ( "net" "net/url" + "strconv" "golang.org/x/net/idna" ) @@ -18,7 +19,17 @@ func TryParseURL(input string) *url.URL { if err == nil { parsedURL.Host = parsedHost } - + // If the port is not present, we need to add it based on the scheme + if parsedURL.Port() == "" { + port := 0 + switch parsedURL.Scheme { + case "http": + port = 80 + case "https": + port = 443 + } + parsedURL.Host = parsedURL.Host + ":" + strconv.Itoa(int(port)) + } host, port, err := net.SplitHostPort(parsedURL.Host) if err == nil { ip := net.ParseIP(host) diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index 8a7b339f8..abac97087 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,6 +11,7 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ + {"http://[0:0:0:0:0:ffff:127.0.0.1]/thefile", "0:0:0:0:0:ffff:127.0.0.1", 80, true}, {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "0000:0000:0000:0000:0000:0000:0000:0001", 4000, true}, {"http://127.0.0.1:8080#\\@127.2.2.2:80/ ", "127.0.0.1", 8080, true}, {"http://1.1.1.1 &@127.0.0.1:4000# @3.3.3.3/", "127.0.0.1", 4000, true}, From d3f66d023f888f0eca3492bf84de3cbcdc71b08e Mon Sep 17 00:00:00 2001 From: root Date: Thu, 14 Aug 2025 13:11:38 +0000 Subject: [PATCH 05/16] update test --- lib/request-processor/helpers/tryParseURL_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/request-processor/helpers/tryParseURL_test.go b/lib/request-processor/helpers/tryParseURL_test.go index 940a57ec6..23e6bfde5 100644 --- a/lib/request-processor/helpers/tryParseURL_test.go +++ b/lib/request-processor/helpers/tryParseURL_test.go @@ -14,7 +14,7 @@ func TestTryParseURL_InvalidURL(t *testing.T) { } func TestTryParseURL_ValidURL(t *testing.T) { - input := "https://example.com" + input := "https://example.com:443" expected, _ := url.Parse(input) result := TryParseURL(input) From 53fb8c78bda09a5bd00aa1f6e2fbcb0de6811481 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 15 Aug 2025 09:16:42 +0000 Subject: [PATCH 06/16] adding user info removal and improving backslash handling in request processing --- .../helpers/normalizeRequestUrl.go | 40 ++++++++++++++++++- .../helpers/normalizeRequestUrl_test.go | 1 + .../ssrf/findHostnameInUserInput.go | 11 ++--- .../ssrf/findHostnameInUserInput_test.go | 2 + 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index f10d8b30f..240da4adf 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -1,6 +1,9 @@ package helpers -import "strings" +import ( + "regexp" + "strings" +) func removeCTLByte(urlStr string) string { for i := 0; i < len(urlStr); i++ { @@ -11,17 +14,50 @@ func removeCTLByte(urlStr string) string { return urlStr } +var backslashAt = regexp.MustCompile(`\\+@`) + // If the urlStr contains \@ we need to replace it with @ // because the URL.Parse will fail to parse the url (invalid userinfo) +// IMPORTANT: there can be multiple backslashes before the @ func removeBackslashAt(urlStr string) string { - return strings.ReplaceAll(urlStr, "\\@", "@") + return backslashAt.ReplaceAllString(urlStr, "@") } func removeWhitespace(urlStr string) string { return strings.ReplaceAll(urlStr, " ", "") } +func removeUserInfo(raw string) string { + schemeEnd := strings.Index(raw, "://") + if schemeEnd == -1 { + // No scheme, can't safely identify authority + return raw + } + + scheme := raw[:schemeEnd+3] + rest := raw[schemeEnd+3:] + + // Authority is up to first '/', '?', or '#' + authorityEnd := len(rest) + for _, sep := range []string{"/", "?", "#"} { + if idx := strings.Index(rest, sep); idx != -1 && idx < authorityEnd { + authorityEnd = idx + } + } + + authority := rest[:authorityEnd] + path := rest[authorityEnd:] + + // Remove userinfo if present (use LAST @) + if at := strings.LastIndex(authority, "@"); at != -1 { + authority = authority[at+1:] + } + + return scheme + authority + path +} + func NormalizeRawUrl(urlStr string) string { + urlStr = removeUserInfo(urlStr) urlStr = removeCTLByte(urlStr) urlStr = removeBackslashAt(urlStr) urlStr = removeWhitespace(urlStr) diff --git a/lib/request-processor/helpers/normalizeRequestUrl_test.go b/lib/request-processor/helpers/normalizeRequestUrl_test.go index 4a5610397..facfdff5a 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl_test.go +++ b/lib/request-processor/helpers/normalizeRequestUrl_test.go @@ -11,6 +11,7 @@ func TestNormalizeRequestUrl(t *testing.T) { {"http://localhost:4000 ", "http://localhost:4000"}, {"http://localhost:4000" + "\x00", "http://localhost:4000"}, {"http://\\@localhost:4000", "http://@localhost:4000"}, + {"http://127.1.1.1:4000\\\\\\@127.0.0.1:80/", "http://127.1.1.1:4000@127.0.0.1:80/"}, } for _, test := range tests { result := NormalizeRawUrl(test.input) diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go index b12e2f65d..6c46edc18 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go @@ -8,17 +8,16 @@ import ( "strings" ) -func getVariants(str string) []string { - variants := []string{str, "http://" + str, "https://" + str} - decodedUserInput, err := url.QueryUnescape(str) - if err == nil && decodedUserInput != str { +func getVariants(userInput string) []string { + variants := []string{userInput, "http://" + userInput, "https://" + userInput} + decodedUserInput, err := url.QueryUnescape(userInput) + if err == nil && decodedUserInput != userInput { variants = append(variants, decodedUserInput, "http://"+decodedUserInput, "https://"+decodedUserInput) } return variants } func findHostnameInUserInput(userInput string, hostname string, port uint32) bool { - userInput = helpers.NormalizeRawUrl(userInput) log.Debugf("findHostnameInUserInput: userInput: %s, hostname: %s, port: %d", userInput, hostname, port) if len(userInput) <= 1 { return false @@ -34,6 +33,8 @@ func findHostnameInUserInput(userInput string, hostname string, port uint32) boo } userInput = helpers.ExtractResourceOrOriginal(userInput) + userInput = helpers.NormalizeRawUrl(userInput) + variants := getVariants(userInput) for _, variant := range variants { diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index abac97087..3d107ec36 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,6 +11,8 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ + {"http://127.1.1.1:4000∖@127.0.0.1:80/", "127.0.0.1", 80, true}, + {"http://127.1.1.1:4000\\\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, {"http://[0:0:0:0:0:ffff:127.0.0.1]/thefile", "0:0:0:0:0:ffff:127.0.0.1", 80, true}, {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "0000:0000:0000:0000:0000:0000:0000:0001", 4000, true}, {"http://127.0.0.1:8080#\\@127.2.2.2:80/ ", "127.0.0.1", 8080, true}, From 976677bed0f6bc6f1dd47c42d2311fb89a557064 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 15 Aug 2025 11:49:34 +0000 Subject: [PATCH 07/16] fix test cases for URL normalization --- lib/request-processor/helpers/normalizeRequestUrl_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/request-processor/helpers/normalizeRequestUrl_test.go b/lib/request-processor/helpers/normalizeRequestUrl_test.go index facfdff5a..7f66d1806 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl_test.go +++ b/lib/request-processor/helpers/normalizeRequestUrl_test.go @@ -10,8 +10,8 @@ func TestNormalizeRequestUrl(t *testing.T) { {"http://localhost:4000", "http://localhost:4000"}, {"http://localhost:4000 ", "http://localhost:4000"}, {"http://localhost:4000" + "\x00", "http://localhost:4000"}, - {"http://\\@localhost:4000", "http://@localhost:4000"}, - {"http://127.1.1.1:4000\\\\\\@127.0.0.1:80/", "http://127.1.1.1:4000@127.0.0.1:80/"}, + {"http://\\@localhost:4000", "http://localhost:4000"}, + {"http://127.1.1.1:4000\\\\\\@127.0.0.1:80/", "http://127.0.0.1:80/"}, } for _, test := range tests { result := NormalizeRawUrl(test.input) From f0bb7ccb7b237c77e8c792d6b3a8a3d98488d730 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 22 Aug 2025 11:30:20 +0000 Subject: [PATCH 08/16] enhance URL normalization and add test cases for SSRF detection --- .../helpers/normalizeRequestUrl.go | 3 ++- .../helpers/normalizeRequestUrl_test.go | 1 + .../ssrf/findHostnameInUserInput_test.go | 1 + tests/cli/ssrf/test_ssrf_obfuscated_host.phpt | 21 +++++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 tests/cli/ssrf/test_ssrf_obfuscated_host.phpt diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index 240da4adf..70762b989 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -57,9 +57,10 @@ func removeUserInfo(raw string) string { } func NormalizeRawUrl(urlStr string) string { - urlStr = removeUserInfo(urlStr) urlStr = removeCTLByte(urlStr) urlStr = removeBackslashAt(urlStr) urlStr = removeWhitespace(urlStr) + urlStr = FixURL(urlStr) + urlStr = removeUserInfo(urlStr) return urlStr } diff --git a/lib/request-processor/helpers/normalizeRequestUrl_test.go b/lib/request-processor/helpers/normalizeRequestUrl_test.go index 7f66d1806..9684d5d9f 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl_test.go +++ b/lib/request-processor/helpers/normalizeRequestUrl_test.go @@ -12,6 +12,7 @@ func TestNormalizeRequestUrl(t *testing.T) { {"http://localhost:4000" + "\x00", "http://localhost:4000"}, {"http://\\@localhost:4000", "http://localhost:4000"}, {"http://127.1.1.1:4000\\\\\\@127.0.0.1:80/", "http://127.0.0.1:80/"}, + {"https:/localhost:4000", "https://localhost:4000"}, } for _, test := range tests { result := NormalizeRawUrl(test.input) diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index 3d107ec36..622d34feb 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,6 +11,7 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ + {"aa:@localhost:8080", "localhost", 8080, true}, {"http://127.1.1.1:4000∖@127.0.0.1:80/", "127.0.0.1", 80, true}, {"http://127.1.1.1:4000\\\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, {"http://[0:0:0:0:0:ffff:127.0.0.1]/thefile", "0:0:0:0:0:ffff:127.0.0.1", 80, true}, diff --git a/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt b/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt new file mode 100644 index 000000000..85972a378 --- /dev/null +++ b/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt @@ -0,0 +1,21 @@ +--TEST-- +Test path ssrf \@ in authority + +--ENV-- +AIKIDO_LOG_LEVEL=INFO +AIKIDO_BLOCK=1 + +--POST-- +test=http://127.1.1.1:4000\@127.0.0.1:80/ + +--FILE-- + + +--EXPECTREGEX-- +.*Fatal error: Uncaught Exception: Aikido firewall has blocked a server-side request forgery.* From a7eb17087a28de0c61fd16ea6d1b6d9d7d2710a6 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 9 Sep 2025 17:28:21 +0000 Subject: [PATCH 09/16] refactor SSRF test --- tests/cli/ssrf/test_ssrf_obfuscated_host.phpt | 55 ++++++++++++++++--- 1 file changed, 48 insertions(+), 7 deletions(-) diff --git a/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt b/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt index 85972a378..519195396 100644 --- a/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt +++ b/tests/cli/ssrf/test_ssrf_obfuscated_host.phpt @@ -1,21 +1,62 @@ --TEST-- -Test path ssrf \@ in authority +Test ssrf - obfuscated host --ENV-- AIKIDO_LOG_LEVEL=INFO AIKIDO_BLOCK=1 --POST-- -test=http://127.1.1.1:4000\@127.0.0.1:80/ +test=http://127.1.1.1:4000\@127.0.0.1:4000 --FILE-- ['pipe', 'r'], + 1 => ['file', '/dev/null', 'a'], + 2 => ['file', '/dev/null', 'a'] +]; + +try { + // Start PHP server + $process = proc_open("php -S $host:$port", $descriptorspec, $pipes); + if (!is_resource($process)) { + throw new RuntimeException("Failed to start PHP server."); + } + + $status = proc_get_status($process); + $pid = $status['pid']; + + // Wait a moment to ensure server starts + sleep(1); + + // Perform the cURL request + $ch1 = curl_init("http://127.0.0.1:4000"); + curl_setopt($ch1, CURLOPT_RETURNTRANSFER, true); + curl_setopt($ch1, CURLOPT_FOLLOWLOCATION, true); + $response = curl_exec($ch1); + curl_close($ch1); + + echo "Response:\n$response\n"; + +} catch (Throwable $e) { + echo "Error: " . $e->getMessage() . "\n"; +} finally { + // Ensure the server is killed if started + if ($pid) { + exec("kill -9 $pid"); + } + if (isset($process) && is_resource($process)) { + proc_close($process); + } +} + + -?> --EXPECTREGEX-- -.*Fatal error: Uncaught Exception: Aikido firewall has blocked a server-side request forgery.* +.*Aikido firewall has blocked a server-side request forgery.* From ed1eed521ec1b0a95cdd65e3b30590dc6645af09 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Sep 2025 06:41:35 +0000 Subject: [PATCH 10/16] update --- lib/agent/go.mod | 8 +++---- lib/agent/go.sum | 8 +++++++ .../helpers/normalizeRequestUrl.go | 21 +++---------------- .../ssrf/findHostnameInUserInput.go | 2 +- .../ssrf/findHostnameInUserInput_test.go | 3 +++ 5 files changed, 19 insertions(+), 23 deletions(-) diff --git a/lib/agent/go.mod b/lib/agent/go.mod index bc6b67d45..702a00c62 100644 --- a/lib/agent/go.mod +++ b/lib/agent/go.mod @@ -6,16 +6,16 @@ toolchain go1.23.3 require ( github.com/stretchr/testify v1.9.0 - google.golang.org/grpc v1.74.2 + google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.6 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.40.0 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + golang.org/x/text v0.26.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/lib/agent/go.sum b/lib/agent/go.sum index 8c68dd312..888cd966b 100644 --- a/lib/agent/go.sum +++ b/lib/agent/go.sum @@ -24,6 +24,8 @@ golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= @@ -40,6 +42,8 @@ golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= @@ -48,6 +52,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= @@ -58,6 +64,8 @@ google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index 70762b989..4385701a3 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -1,32 +1,19 @@ package helpers import ( - "regexp" "strings" ) +// remove all control characters (< 32) and 0x7f(DEL) + whitespace func removeCTLByte(urlStr string) string { for i := 0; i < len(urlStr); i++ { - if urlStr[i] < ' ' || urlStr[i] == 0x7f { + if urlStr[i] <= ' ' || urlStr[i] == 0x7f { urlStr = urlStr[:i] + urlStr[i+1:] } } return urlStr } -var backslashAt = regexp.MustCompile(`\\+@`) - -// If the urlStr contains \@ we need to replace it with @ -// because the URL.Parse will fail to parse the url (invalid userinfo) -// IMPORTANT: there can be multiple backslashes before the @ -func removeBackslashAt(urlStr string) string { - return backslashAt.ReplaceAllString(urlStr, "@") -} - -func removeWhitespace(urlStr string) string { - return strings.ReplaceAll(urlStr, " ", "") -} - func removeUserInfo(raw string) string { schemeEnd := strings.Index(raw, "://") if schemeEnd == -1 { @@ -37,7 +24,7 @@ func removeUserInfo(raw string) string { scheme := raw[:schemeEnd+3] rest := raw[schemeEnd+3:] - // Authority is up to first '/', '?', or '#' + // Authority is up to first '/', '?', or '#' (https://datatracker.ietf.org/doc/html/rfc3986#section-3.2) authorityEnd := len(rest) for _, sep := range []string{"/", "?", "#"} { if idx := strings.Index(rest, sep); idx != -1 && idx < authorityEnd { @@ -58,8 +45,6 @@ func removeUserInfo(raw string) string { func NormalizeRawUrl(urlStr string) string { urlStr = removeCTLByte(urlStr) - urlStr = removeBackslashAt(urlStr) - urlStr = removeWhitespace(urlStr) urlStr = FixURL(urlStr) urlStr = removeUserInfo(urlStr) return urlStr diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go index 6c46edc18..f5c4d90ac 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput.go @@ -45,7 +45,7 @@ func findHostnameInUserInput(userInput string, hostname string, port uint32) boo // https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 // "The host subcomponent is case-insensitive." - if userInputURL != nil && strings.EqualFold(userInputURL.Hostname(), hostnameURL.Hostname()) { + if strings.EqualFold(userInputURL.Hostname(), hostnameURL.Hostname()) { userPort := helpers.GetPortFromURL(userInputURL) if port == 0 { diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index 622d34feb..9b186d8e0 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,8 +11,10 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ + {"http://127.1.1.1:4000\\@127.0.0.1:8080\\", "127.0.0.1", 8080, true}, {"aa:@localhost:8080", "localhost", 8080, true}, {"http://127.1.1.1:4000∖@127.0.0.1:80/", "127.0.0.1", 80, true}, + {"http://127.1.1.1:4000∖@127.0.0.1:80\\", "127.0.0.1", 80, true}, {"http://127.1.1.1:4000\\\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, {"http://[0:0:0:0:0:ffff:127.0.0.1]/thefile", "0:0:0:0:0:ffff:127.0.0.1", 80, true}, {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "0000:0000:0000:0000:0000:0000:0000:0001", 4000, true}, @@ -20,6 +22,7 @@ func TestFindHostnameInUserInput(t *testing.T) { {"http://1.1.1.1 &@127.0.0.1:4000# @3.3.3.3/", "127.0.0.1", 4000, true}, {"http://127.1.1.1:4000:\\@@127.0.0.1:8080/", "127.0.0.1", 8080, true}, {"http://127.1.1.1:4000\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, + {"http://127.0.0.1:8080/@/127.1.1.1:4000", "127.0.0.1", 8080, true}, {"http://%31%32%37.%30.%30.%31:4000", "127.0.0.1", 4000, true}, {"http://%30:4000", "0", 4000, true}, {"http://127%2E0%2E0%2E1:4000", "127.0.0.1", 4000, true}, From e13b0622f4a09f797a360ab2a0aad42a7d7b7dbb Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Sep 2025 06:48:19 +0000 Subject: [PATCH 11/16] update --- .../vulnerabilities/ssrf/findHostnameInUserInput_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go index 9b186d8e0..91229bb9b 100644 --- a/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/findHostnameInUserInput_test.go @@ -11,10 +11,8 @@ func TestFindHostnameInUserInput(t *testing.T) { port uint32 expected bool }{ - {"http://127.1.1.1:4000\\@127.0.0.1:8080\\", "127.0.0.1", 8080, true}, {"aa:@localhost:8080", "localhost", 8080, true}, {"http://127.1.1.1:4000∖@127.0.0.1:80/", "127.0.0.1", 80, true}, - {"http://127.1.1.1:4000∖@127.0.0.1:80\\", "127.0.0.1", 80, true}, {"http://127.1.1.1:4000\\\\@127.0.0.1:8080/", "127.0.0.1", 8080, true}, {"http://[0:0:0:0:0:ffff:127.0.0.1]/thefile", "0:0:0:0:0:ffff:127.0.0.1", 80, true}, {"http://[0000:0000:0000:0000:0000:0000:0000:0001]:4000", "0000:0000:0000:0000:0000:0000:0000:0001", 4000, true}, From 95cfe5f120c67285684d210c6ffced4dc7724afb Mon Sep 17 00:00:00 2001 From: root Date: Wed, 10 Sep 2025 10:00:31 +0000 Subject: [PATCH 12/16] add comment to clarify URL parsing error handling --- lib/request-processor/context/event_getters.go | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/request-processor/context/event_getters.go b/lib/request-processor/context/event_getters.go index 07c885e71..5492bef0b 100644 --- a/lib/request-processor/context/event_getters.go +++ b/lib/request-processor/context/event_getters.go @@ -50,6 +50,7 @@ func GetModule() string { func getHostNameAndPort(urlCallbackId int) (string, uint32) { // urlcallbackid is the type of data we request, eg C.OUTGOING_REQUEST_URL urlStr := Context.Callback(urlCallbackId) // remove all control characters (< 32) and 0x7f(DEL) also replace \@ with @ and remove all whitespace + // url.Parse fails if the url contains control characters urlStr = helpers.NormalizeRawUrl(urlStr) urlParsed, err := url.Parse(urlStr) if err != nil { From 70d5d3d2d6eeb34887ddfe018cd526222e330739 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 11 Sep 2025 10:38:12 +0000 Subject: [PATCH 13/16] test --- lib/request-processor/helpers/normalizeRequestUrl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index 4385701a3..da492ab97 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -35,7 +35,7 @@ func removeUserInfo(raw string) string { authority := rest[:authorityEnd] path := rest[authorityEnd:] - // Remove userinfo if present (use LAST @) + // Remove userinfo if present if at := strings.LastIndex(authority, "@"); at != -1 { authority = authority[at+1:] } From 0ef59f95e51b521bf932e9309144c91b1138d257 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 22 Sep 2025 14:56:55 +0000 Subject: [PATCH 14/16] implement URL unescaping in normalization process --- lib/agent/go.mod | 2 +- lib/agent/go.sum | 4 ++-- lib/request-processor/helpers/normalizeRequestUrl.go | 10 ++++++++++ .../helpers/normalizeRequestUrl_test.go | 1 + 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/agent/go.mod b/lib/agent/go.mod index 702a00c62..aa5de276e 100644 --- a/lib/agent/go.mod +++ b/lib/agent/go.mod @@ -6,7 +6,7 @@ toolchain go1.23.3 require ( github.com/stretchr/testify v1.9.0 - google.golang.org/grpc v1.75.0 + google.golang.org/grpc v1.75.1 google.golang.org/protobuf v1.36.6 ) diff --git a/lib/agent/go.sum b/lib/agent/go.sum index 888cd966b..9ef849ffd 100644 --- a/lib/agent/go.sum +++ b/lib/agent/go.sum @@ -64,8 +64,8 @@ google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= -google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= -google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index da492ab97..c555433ed 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -1,6 +1,7 @@ package helpers import ( + "net/url" "strings" ) @@ -43,7 +44,16 @@ func removeUserInfo(raw string) string { return scheme + authority + path } +func UnescapeUrl(urlStr string) string { + unescapedUrl, err := url.QueryUnescape(urlStr) + if err != nil { + return urlStr + } + return unescapedUrl +} + func NormalizeRawUrl(urlStr string) string { + urlStr = UnescapeUrl(urlStr) urlStr = removeCTLByte(urlStr) urlStr = FixURL(urlStr) urlStr = removeUserInfo(urlStr) diff --git a/lib/request-processor/helpers/normalizeRequestUrl_test.go b/lib/request-processor/helpers/normalizeRequestUrl_test.go index 9684d5d9f..a179e5320 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl_test.go +++ b/lib/request-processor/helpers/normalizeRequestUrl_test.go @@ -13,6 +13,7 @@ func TestNormalizeRequestUrl(t *testing.T) { {"http://\\@localhost:4000", "http://localhost:4000"}, {"http://127.1.1.1:4000\\\\\\@127.0.0.1:80/", "http://127.0.0.1:80/"}, {"https:/localhost:4000", "https://localhost:4000"}, + {"http://127%2E0%2E0%2E1:4000", "http://127.0.0.1:4000"}, } for _, test := range tests { result := NormalizeRawUrl(test.input) From 9e0a174e80454dd285c5b72b221c5d1cea88e254 Mon Sep 17 00:00:00 2001 From: Marian Popovici Date: Tue, 4 Nov 2025 10:38:40 +0200 Subject: [PATCH 15/16] update Go module --- lib/agent/go.mod | 16 ++++++++-------- lib/agent/go.sum | 12 +++++++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/lib/agent/go.mod b/lib/agent/go.mod index aa5de276e..2bbd9ba12 100644 --- a/lib/agent/go.mod +++ b/lib/agent/go.mod @@ -1,21 +1,21 @@ module main -go 1.23.0 +go 1.24.0 -toolchain go1.23.3 +toolchain go1.24.8 require ( github.com/stretchr/testify v1.9.0 - google.golang.org/grpc v1.75.1 + google.golang.org/grpc v1.76.0 google.golang.org/protobuf v1.36.6 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect gopkg.in/yaml.v3 v3.0.1 // indirect -) +) \ No newline at end of file diff --git a/lib/agent/go.sum b/lib/agent/go.sum index 9ef849ffd..dbf5b3672 100644 --- a/lib/agent/go.sum +++ b/lib/agent/go.sum @@ -26,6 +26,8 @@ golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= @@ -34,6 +36,8 @@ golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= @@ -44,6 +48,8 @@ golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= @@ -54,6 +60,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= @@ -66,6 +74,8 @@ google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= @@ -75,4 +85,4 @@ google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= \ No newline at end of file From 04c34e221e32d32b70838c509dfe6b4cae1380a0 Mon Sep 17 00:00:00 2001 From: Marian Popovici Date: Tue, 4 Nov 2025 11:09:21 +0200 Subject: [PATCH 16/16] implement conversion of IPv6-mapped IPv4 addresses in URL normalization --- .../helpers/normalizeRequestUrl.go | 29 +++++++++++++++++++ .../ssrf/containsPrivateIPAddress_test.go | 2 -- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/lib/request-processor/helpers/normalizeRequestUrl.go b/lib/request-processor/helpers/normalizeRequestUrl.go index c555433ed..fe35b1a30 100644 --- a/lib/request-processor/helpers/normalizeRequestUrl.go +++ b/lib/request-processor/helpers/normalizeRequestUrl.go @@ -2,6 +2,7 @@ package helpers import ( "net/url" + "regexp" "strings" ) @@ -52,10 +53,38 @@ func UnescapeUrl(urlStr string) string { return unescapedUrl } +// ConvertIPv6Mapped converts IPv6-mapped IPv4 (only if it contains ::ffff:) +// Example: "http://[::ffff:10.0.0.1]" -> "http://10.0.0.1" +func convertIPv6Mapped(input string) string { + // Return immediately if not IPv6-mapped form + if !strings.Contains(input, "::ffff:") { + return input + } + + // Extract URL scheme if present (http://, https://, etc.) + scheme := "" + if strings.Contains(input, "://") { + parts := strings.SplitN(input, "://", 2) + scheme = parts[0] + "://" + input = parts[1] + } + + // Strip brackets + input = strings.TrimPrefix(input, "[") + input = strings.TrimSuffix(input, "]") + + // Replace ::ffff:x.x.x.x -> x.x.x.x + re := regexp.MustCompile(`::ffff:(\d+\.\d+\.\d+\.\d+)`) + ip := re.ReplaceAllString(input, "$1") + + return scheme + ip +} + func NormalizeRawUrl(urlStr string) string { urlStr = UnescapeUrl(urlStr) urlStr = removeCTLByte(urlStr) urlStr = FixURL(urlStr) urlStr = removeUserInfo(urlStr) + urlStr = convertIPv6Mapped(urlStr) return urlStr } diff --git a/lib/request-processor/vulnerabilities/ssrf/containsPrivateIPAddress_test.go b/lib/request-processor/vulnerabilities/ssrf/containsPrivateIPAddress_test.go index 84339124d..d7ddc6e1a 100644 --- a/lib/request-processor/vulnerabilities/ssrf/containsPrivateIPAddress_test.go +++ b/lib/request-processor/vulnerabilities/ssrf/containsPrivateIPAddress_test.go @@ -142,8 +142,6 @@ var privateIPs = []string{ "0000:0000:0000:0000:0000:0000:0000:0000", "::", "::1", - "::ffff:0.0.0.0", - "::ffff:127.0.0.1", "fe80::", "fe80::1", "fe80::abc:1",