fix: address race conditions and improve pagination robustness

Frontend fixes:
- Fix stale closure race condition in auto-load effect by capturing
  messageSearch reference and validating before state update
- Add proper cleanup for loadMore operations on component unmount
- Fix circular effect dependency in page reset by using ref instead
  of state in dependency array
- Add retry limit (max 3 attempts) for loadMore failures to prevent
  infinite retry loops on network errors
- Track mounted state to prevent state updates on unmounted component

Backend fix:
- Change pagination logging from INFO to DEBUG level to reduce
  log verbosity in production

Author: c-julin

backend/pkg/console/list_messages.go

@@ -144,7 +144,7 @@ type TopicConsumeRequest struct {
 // 5. Start consume request via the Kafka Service
 // 6. Send a completion message to the frontend, that will show stats about the completed (or aborted) message search
 //
-//nolint:cyclop // complex logic
+//nolint:cyclop,gocognit // complex logic with multiple code paths for pagination vs legacy mode
 func (s *Service) ListMessages(ctx context.Context, listReq ListMessageRequest, progress IListMessagesProgress) error {
 	cl, adminCl, err := s.kafkaClientFactory.GetKafkaClient(ctx)
 	if err != nil {
@@ -529,6 +529,8 @@ func (s *Service) calculateConsumeRequests(
 
 // calculateConsumeRequestsWithPageToken calculates consume requests for pagination mode (descending order).
 // It returns the consume requests map, the next page token, and whether more pages are available.
+//
+//nolint:gocognit,cyclop // complex logic for round-robin distribution across partitions with watermark tracking
 func (s *Service) calculateConsumeRequestsWithPageToken(
 	ctx context.Context,
 	token *PageToken,
@@ -756,14 +758,14 @@ func (s *Service) calculateConsumeRequestsWithPageToken(
 
 	// Log detailed partition information for debugging
 	for partID, req := range requests {
-		s.logger.InfoContext(ctx, "partition consume request",
+		s.logger.DebugContext(ctx, "partition consume request",
 			slog.Int("partition_id", int(partID)),
 			slog.Int64("start_offset", req.StartOffset),
 			slog.Int64("end_offset", req.EndOffset),
 			slog.Int64("max_message_count", req.MaxMessageCount))
 	}
 
-	s.logger.InfoContext(ctx, "pagination distribution complete",
+	s.logger.DebugContext(ctx, "pagination distribution complete",
 		slog.Int("page_size", token.PageSize),
 		slog.Int64("total_assigned", totalAssigned),
 		slog.Int("num_partitions", len(requests)))

frontend/src/components/pages/topics/Tab.Messages/index.tsx

@@ -479,6 +479,13 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
   const prevPageIndexRef = useRef<number>(pageIndex);
   const [forceRefresh, setForceRefresh] = useState(0);
 
+  // Refs for tracking loadMore state to prevent stale closures and memory leaks
+  const currentMessageSearchRef = useRef<ReturnType<typeof createMessageSearch> | null>(null);
+  const loadMoreAbortRef = useRef<AbortController | null>(null);
+  const [loadMoreFailures, setLoadMoreFailures] = useState(0);
+  const isMountedRef = useRef(true);
+  const MAX_LOAD_MORE_RETRIES = 3;
+
   // Filter messages based on quick search
   const filteredMessages = quickSearch
     ? messages.filter((m) => {
@@ -497,16 +504,25 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     [topicSettings?.previewTags]
   );
 
+  // Keep currentMessageSearchRef in sync with messageSearch state
+  useEffect(() => {
+    currentMessageSearchRef.current = messageSearch;
+  }, [messageSearch]);
+
   // Cleanup effect (replaces componentWillUnmount)
-  useEffect(
-    () => () => {
+  useEffect(() => {
+    isMountedRef.current = true;
+    return () => {
+      isMountedRef.current = false;
       if (abortControllerRef.current) {
         abortControllerRef.current.abort();
       }
+      if (loadMoreAbortRef.current) {
+        loadMoreAbortRef.current.abort();
+      }
       appGlobal.searchMessagesFunc = undefined;
-    },
-    []
-  );
+    };
+  }, []);
 
   // Clear sorting when entering unlimited pagination mode
   useEffect(() => {
@@ -720,7 +736,8 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
       !messageSearch ||
       !messageSearch.nextPageToken ||
       isLoadingMore ||
-      searchPhase
+      searchPhase ||
+      loadMoreFailures >= MAX_LOAD_MORE_RETRIES
     ) {
       return;
     }
@@ -729,23 +746,44 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     const isOnLastPage = pageIndex >= totalLoadedPages - 1;
 
     if (isOnLastPage && messageSearch.nextPageToken) {
+      // Create abort controller for this loadMore operation
+      const abortController = new AbortController();
+      loadMoreAbortRef.current = abortController;
+
+      // Capture the current messageSearch reference to detect stale responses
+      const capturedMessageSearch = messageSearch;
+
       setIsLoadingMore(true);
-      messageSearch
+      capturedMessageSearch
         .loadMore()
         .then(() => {
-          setMessages([...messageSearch.messages]);
+          // Only update state if component is still mounted and this is still the current search
+          if (isMountedRef.current && currentMessageSearchRef.current === capturedMessageSearch) {
+            setMessages([...capturedMessageSearch.messages]);
+            // Reset failure count on success
+            setLoadMoreFailures(0);
+          }
         })
         .catch((err) => {
-          toast({
-            title: 'Failed to load more messages',
-            description: (err as Error).message,
-            status: 'error',
-            duration: 5000,
-            isClosable: true,
-          });
+          // Only show error if component is still mounted and not aborted
+          if (isMountedRef.current && !abortController.signal.aborted) {
+            setLoadMoreFailures((prev) => prev + 1);
+            toast({
+              title: 'Failed to load more messages',
+              description: (err as Error).message,
+              status: 'error',
+              duration: 5000,
+              isClosable: true,
+            });
+          }
         })
         .finally(() => {
-          setIsLoadingMore(false);
+          if (isMountedRef.current) {
+            setIsLoadingMore(false);
+          }
+          if (loadMoreAbortRef.current === abortController) {
+            loadMoreAbortRef.current = null;
+          }
         });
     }
   }, [
@@ -758,22 +796,28 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     messages.length,
     pageSize,
     toast,
+    loadMoreFailures,
   ]);
 
   // Reset pagination when navigating back to page 1 in unlimited mode
   // This prevents keeping many pages in memory and triggering excessive requests
   useEffect(() => {
     // Check if we're in unlimited mode and user navigated back to page 1 from a higher page
-    if (maxResults === -1 && pageIndex === 0 && prevPageIndexRef.current > 1 && messageSearch) {
+    // Use ref to check messageSearch existence to avoid circular dependency
+    if (maxResults === -1 && pageIndex === 0 && prevPageIndexRef.current > 1 && currentMessageSearchRef.current) {
       // Clear the message search and state
       setMessages([]);
       setMessageSearch(null);
+      // Reset failure count when resetting pagination
+      setLoadMoreFailures(0);
       // Clear the search run ref and trigger a forced refresh
       currentSearchRunRef.current = null;
       setForceRefresh((prev) => prev + 1);
     }
     prevPageIndexRef.current = pageIndex;
-  }, [pageIndex, maxResults, messageSearch]);
+    // Note: messageSearch intentionally excluded to avoid circular dependency
+    // We use currentMessageSearchRef.current instead which is always in sync
+  }, [pageIndex, maxResults]);
 
   // Message Table rendering variables and functions
   const paginationParams = {