feat(proxy): Include a X-Cache Header that indicates whether chproxy had a cache miss or hit. (#313)

X-Cache will be set to HIT if a response came from the Cache, otherwise it will be set to MISS.

Fixes #288

Signed-off-by: Lennard Eijsackers <[email protected]>

Author: Blokje5

docs/content/en/configuration/caching.md

@@ -62,3 +62,9 @@ User Y will get the cached response from user X's query.
 
 Since 1.20.0, the cache is specific for each user by default since it's better in terms of security.
 It's possible to use the previous behavior by setting the following property of the cache in the config file `shared_with_all_users = true` 
+
+#### Detecting Cache Hits
+
+`Chproxy` will respond with an `X-Cache` header with a value of `HIT` if it returned a response from either the local or the distributed cache. Otherwise `X-Cache` will be set to `MISS`. 
+If the response couldn't be cached due to the configuration (e.g. a payload that is too large), `N/A` will be returned. This can be used for example to determine 
+whether the ClickHouse query stats in the response can be trusted or are cached responses.

io.go

@@ -44,7 +44,13 @@ type statResponseWriter struct {
 	bytesWritten prometheus.Counter
 }
 
-func RespondWithData(rw http.ResponseWriter, data io.Reader, metadata cache.ContentMetadata, ttl time.Duration, statusCode int, labels prometheus.Labels) error {
+const (
+	XCacheHit  = "HIT"
+	XCacheMiss = "MISS"
+	XCacheNA   = "N/A"
+)
+
+func RespondWithData(rw http.ResponseWriter, data io.Reader, metadata cache.ContentMetadata, ttl time.Duration, cacheHit string, statusCode int, labels prometheus.Labels) error {
 	h := rw.Header()
 	if len(metadata.Type) > 0 {
 		h.Set("Content-Type", metadata.Type)
@@ -59,6 +65,9 @@ func RespondWithData(rw http.ResponseWriter, data io.Reader, metadata cache.Cont
 		expireSeconds := uint(ttl / time.Second)
 		h.Set("Cache-Control", fmt.Sprintf("max-age=%d", expireSeconds))
 	}
+
+	h.Set("X-Cache", cacheHit)
+
 	rw.WriteHeader(statusCode)
 
 	if _, err := io.Copy(rw, data); err != nil {

main_test.go

@@ -99,6 +99,7 @@ func TestServe(t *testing.T) {
 					t.Fatalf("unexpected status code: %d; expected: %d", resp.StatusCode, http.StatusOK)
 				}
 				checkResponse(t, resp.Body, expectedOkResp)
+				checkHeader(t, resp, "X-Cache", "MISS")
 
 				// check cached response
 				credHash, _ := calcCredentialHash("default", "qwerty")
@@ -120,11 +121,12 @@ func TestServe(t *testing.T) {
 				if err != nil {
 					t.Fatalf("unexpected error while getting response from cache: %s", err)
 				}
-				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, 200, labels)
+				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, XCacheHit, 200, labels)
 				if err != nil {
 					t.Fatalf("unexpected error while getting response from cache: %s", err)
 				}
 				checkResponse(t, rw.Body, expectedOkResp)
+				checkHeader(t, rw.Result(), "X-Cache", XCacheHit)
 			},
 			startTLS,
 		},
@@ -145,6 +147,7 @@ func TestServe(t *testing.T) {
 				}
 
 				checkResponse(t, resp.Body, expectedOkResp)
+				checkHeader(t, resp, "X-Cache", XCacheNA)
 
 				key := &cache.Key{
 					Query:          []byte(q),
@@ -198,7 +201,7 @@ func TestServe(t *testing.T) {
 					t.Fatalf("unexpected error while getting response from cache: %s", err)
 				}
 
-				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, 200, labels)
+				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, XCacheHit, 200, labels)
 				if err != nil {
 					t.Fatalf("unexpected error while getting response from cache: %s", err)
 				}
@@ -250,7 +253,7 @@ func TestServe(t *testing.T) {
 					t.Fatalf("unexpected error while writing reposnse from cache: %s", err)
 				}
 
-				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, 200, labels)
+				err = RespondWithData(rw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, XCacheMiss, 200, labels)
 				if err != nil {
 					t.Fatalf("unexpected error while getting response from cache: %s", err)
 				}
@@ -1183,6 +1186,17 @@ func httpGet(t *testing.T, url string, statusCode int) *http.Response {
 	return resp
 }
 
+func checkHeader(t *testing.T, resp *http.Response, header string, expected string) {
+	t.Helper()
+
+	h := resp.Header
+	v := h.Get(header)
+
+	if v != expected {
+		t.Fatalf("for header: %s got: %s, expected %s", header, v, expected)
+	}
+}
+
 func httpRequest(t *testing.T, request *http.Request, statusCode int) (*http.Response, error) {
 	t.Helper()
 	client := http.Client{}

proxy.go

@@ -357,7 +357,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 		cacheHit.With(labels).Inc()
 		cachedResponseDuration.With(labels).Observe(time.Since(startTime).Seconds())
 		log.Debugf("%s: cache hit", s)
-		_ = RespondWithData(srw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, http.StatusOK, labels)
+		_ = RespondWithData(srw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, XCacheHit, http.StatusOK, labels)
 		return
 	}
 	// Await for potential result from concurrent query
@@ -370,7 +370,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 			cachedData, err := userCache.Get(key)
 			if err == nil {
 				defer cachedData.Data.Close()
-				_ = RespondWithData(srw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, http.StatusOK, labels)
+				_ = RespondWithData(srw, cachedData.Data, cachedData.ContentMetadata, cachedData.Ttl, XCacheHit, http.StatusOK, labels)
 				cacheHitFromConcurrentQueries.With(labels).Inc()
 				log.Debugf("%s: cache hit after awaiting concurrent query", s)
 				return
@@ -444,7 +444,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 			return
 		}
 
-		err = RespondWithData(srw, reader, contentMetadata, 0*time.Second, statusCode, labels)
+		err = RespondWithData(srw, reader, contentMetadata, 0*time.Second, XCacheMiss, statusCode, labels)
 		if err != nil {
 			err = fmt.Errorf("%s: %w; query: %q", s, err, q)
 			respondWith(srw, err, http.StatusInternalServerError)
@@ -457,7 +457,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 
 			rp.completeTransaction(s, statusCode, userCache, key, q, "")
 
-			err = RespondWithData(srw, reader, contentMetadata, 0*time.Second, tmpFileRespWriter.StatusCode(), labels)
+			err = RespondWithData(srw, reader, contentMetadata, 0*time.Second, XCacheNA, tmpFileRespWriter.StatusCode(), labels)
 			if err != nil {
 				err = fmt.Errorf("%s: %w; query: %q", s, err, q)
 				respondWith(srw, err, http.StatusInternalServerError)
@@ -481,7 +481,7 @@ func (rp *reverseProxy) serveFromCache(s *scope, srw *statResponseWriter, req *h
 			respondWith(srw, err, http.StatusInternalServerError)
 			return
 		}
-		err = RespondWithData(srw, reader, contentMetadata, expiration, statusCode, labels)
+		err = RespondWithData(srw, reader, contentMetadata, expiration, XCacheMiss, statusCode, labels)
 		if err != nil {
 			err = fmt.Errorf("%s: %w; query: %q", s, err, q)
 			respondWith(srw, err, http.StatusInternalServerError)