From e405a42aa2f20f19c79828581c21cdfb64ca5757 Mon Sep 17 00:00:00 2001
From: Anas Khan <83116240+anxkhn@users.noreply.github.com>
Date: Fri, 3 Jul 2026 12:43:12 +0530
Subject: [PATCH] Close querier in remote read handler to fix querier leak

The remote read handler creates a storage.Querier for each sub-query in
its own goroutine but never closes it, so every remote read sub-query
leaked its querier and the resources it holds (inflight-query trackers,
store-gateway clients, index readers) for the lifetime of the process.
A client issuing many remote reads (for example Prometheus federation,
Thanos sidecar remote_read, or Grafana) steadily accumulated leaked
queriers.

Defer querier.Close() inside the goroutine so it runs on every path,
matching every other querier call site in the codebase.

Add a regression test asserting that each per-query querier is closed.

Signed-off-by: Anas Khan <83116240+anxkhn@users.noreply.github.com>
---
 CHANGELOG.md                    |  1 +
 pkg/querier/remote_read.go      |  1 +
 pkg/querier/remote_read_test.go | 40 +++++++++++++++++++++++++++++++++
 3 files changed, 42 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3a9c2e68ece..a80b0af79c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -77,6 +77,7 @@
 * [BUGFIX] Querier: Fix panic due to request tracker truncating multi-byte UTF-8 character #7640
 * [BUGFIX] Ingester: Fix panic (`HistogramProtoToHistogram called with a float histogram`) when ingesting a float native histogram with a zero count (e.g. a staleness marker or empty histogram). The decoder is now selected by histogram type via `IsFloatHistogram()` instead of by count value. #7645
 * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638
+* [BUGFIX] Querier: Fix querier leak in the remote read handler by closing the per-query `storage.Querier` once each sub-query completes. Previously every remote read sub-query leaked its querier and the resources it held (inflight-query trackers, store-gateway clients, index readers) for the lifetime of the process. #7670
 
 ## 1.21.0 2026-04-24
 
diff --git a/pkg/querier/remote_read.go b/pkg/querier/remote_read.go
index 5067f1a8f51..cfa8efab155 100644
--- a/pkg/querier/remote_read.go
+++ b/pkg/querier/remote_read.go
@@ -46,6 +46,7 @@ func RemoteReadHandler(q storage.Queryable, logger log.Logger) http.Handler {
 					errors <- err
 					return
 				}
+				defer querier.Close()
 
 				params := &storage.SelectHints{
 					Start: int64(from),
diff --git a/pkg/querier/remote_read_test.go b/pkg/querier/remote_read_test.go
index b3e7c0c28b4..1707674505d 100644
--- a/pkg/querier/remote_read_test.go
+++ b/pkg/querier/remote_read_test.go
@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"sync/atomic"
 	"testing"
 
 	"github.com/go-kit/log"
@@ -88,6 +89,33 @@ func TestRemoteReadHandler(t *testing.T) {
 	require.Equal(t, expected, response)
 }
 
+func TestRemoteReadHandler_Closes_Querier(t *testing.T) {
+	t.Parallel()
+	var closed atomic.Int64
+	q := storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) {
+		return &closeCountingQuerier{closed: &closed}, nil
+	})
+	handler := RemoteReadHandler(q, log.NewNopLogger())
+
+	const numQueries = 3
+	queries := make([]*client.QueryRequest, numQueries)
+	for i := range queries {
+		queries[i] = &client.QueryRequest{StartTimestampMs: 0, EndTimestampMs: 10}
+	}
+	requestBody, err := proto.Marshal(&client.ReadRequest{Queries: queries})
+	require.NoError(t, err)
+	requestBody = snappy.Encode(nil, requestBody)
+	request, err := http.NewRequest("GET", "/query", bytes.NewReader(requestBody))
+	require.NoError(t, err)
+	request.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0")
+
+	recorder := httptest.NewRecorder()
+	handler.ServeHTTP(recorder, request)
+
+	require.Equal(t, 200, recorder.Result().StatusCode)
+	require.Equal(t, int64(numQueries), closed.Load(), "each per-query querier must be closed")
+}
+
 type mockQuerier struct {
 	matrix model.Matrix
 }
@@ -110,3 +138,15 @@ func (m mockQuerier) LabelNames(ctx context.Context, _ *storage.LabelHints, matc
 func (mockQuerier) Close() error {
 	return nil
 }
+
+// closeCountingQuerier records how many times Close is called so a test can
+// assert the remote read handler releases every querier it creates.
+type closeCountingQuerier struct {
+	mockQuerier
+	closed *atomic.Int64
+}
+
+func (q *closeCountingQuerier) Close() error {
+	q.closed.Add(1)
+	return nil
+}