feat: add unlimited pagination UI

- Add pageToken/nextPageToken to message search state
- Auto-load next page when user reaches last loaded page
- Reset pagination when navigating back to page 1
- Disable sorting in unlimited mode (maxResults=-1)
- Hide unlimited option in Latest/Live mode
- Auto-reset to 50 when switching to Latest/Live from unlimited

Author: c-julin

backend/pkg/console/page_token_test.go

@@ -1,4 +1,4 @@
-// Copyright 2022 Redpanda Data, Inc.
+// Copyright 2026 Redpanda Data, Inc.
 //
 // Use of this software is governed by the Business Source License
 // included in the file https://github.com/redpanda-data/redpanda/blob/dev/licenses/bsl.md

frontend/src/components/pages/topics/Tab.Messages/index.tsx

@@ -40,6 +40,7 @@ import {
   MenuDivider,
   MenuItem,
   MenuList,
+  Spinner,
   Tooltip,
   useBreakpoint,
   useToast,
@@ -194,6 +195,22 @@ const defaultSelectChakraStyles = {
   }),
 } as const;
 
+const maxResultsSelectChakraStyles = {
+  control: (provided: Record<string, unknown>) => ({
+    ...provided,
+    minWidth: '140px', // Ensure enough width for "∞ Unlimited"
+  }),
+  option: (provided: Record<string, unknown>) => ({
+    ...provided,
+    wordBreak: 'keep-all',
+    whiteSpace: 'nowrap',
+  }),
+  menuList: (provided: Record<string, unknown>) => ({
+    ...provided,
+    minWidth: '140px', // Match control width
+  }),
+} as const;
+
 const inlineSelectChakraStyles = {
   ...defaultSelectChakraStyles,
   control: (provided: Record<string, unknown>) => ({
@@ -451,8 +468,16 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
   const [totalMessagesConsumed, setTotalMessagesConsumed] = useState(0);
   const [elapsedMs, setElapsedMs] = useState<number | null>(null);
 
+  // Pagination state
+  const [messageSearch, setMessageSearch] = useState<ReturnType<typeof createMessageSearch> | null>(null);
+  const [isLoadingMore, setIsLoadingMore] = useState(false);
+
   const currentSearchRunRef = useRef<string | null>(null);
   const abortControllerRef = useRef<AbortController | null>(null);
+  const prevStartOffsetRef = useRef<number>(startOffset);
+  const prevMaxResultsRef = useRef<number>(maxResults);
+  const prevPageIndexRef = useRef<number>(pageIndex);
+  const [forceRefresh, setForceRefresh] = useState(0);
 
   // Filter messages based on quick search
   const filteredMessages = quickSearch
@@ -483,6 +508,46 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     []
   );
 
+  // Clear sorting when entering unlimited pagination mode
+  useEffect(() => {
+    if (maxResults === -1 && sorting.length > 0) {
+      setSortingState([]);
+    }
+  }, [maxResults, sorting.length, setSortingState]);
+
+  // Reset to page 1 when start offset changes (e.g., switching from Newest to Beginning)
+  useEffect(() => {
+    // Only reset if startOffset actually changed (not on initial mount or re-renders)
+    if (prevStartOffsetRef.current !== startOffset) {
+      setPageIndex(0);
+      prevStartOffsetRef.current = startOffset;
+    }
+  }, [startOffset, setPageIndex]);
+
+  // Reset to page 1 when max results changes (e.g., switching from Unlimited to fixed size)
+  useEffect(() => {
+    // Only reset if maxResults actually changed (not on initial mount or re-renders)
+    if (prevMaxResultsRef.current !== maxResults) {
+      setPageIndex(0);
+      prevMaxResultsRef.current = maxResults;
+    }
+  }, [maxResults, setPageIndex]);
+
+  // Reset maxResults to 50 when switching to Latest/Live from unlimited
+  const prevStartOffsetForMaxResultsRef = useRef<number>(startOffset);
+  useEffect(() => {
+    // Only reset if we just switched TO Latest/Live (not already on it)
+    const justSwitchedToLatestLive =
+      prevStartOffsetForMaxResultsRef.current !== PartitionOffsetOrigin.End &&
+      startOffset === PartitionOffsetOrigin.End;
+
+    if (justSwitchedToLatestLive && maxResults === -1) {
+      setMaxResults(50);
+    }
+
+    prevStartOffsetForMaxResultsRef.current = startOffset;
+  }, [startOffset, maxResults, setMaxResults]);
+
   // Convert executeMessageSearch to useCallback
   // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: complex business logic
   const executeMessageSearch = useCallback(
@@ -520,12 +585,17 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
         }
       }
 
-      const request = {
-        topicName: props.topic.topicName,
-        partitionId: partitionID,
-        startOffset,
-        startTimestamp: currentSearchParams?.startTimestamp ?? uiState.topicSettings.searchParams.startTimestamp,
-        maxResults,
+      // Calculate backend page size: for pagination mode (maxResults === -1),
+    // fetch exactly 1 page at a time to handle compaction gaps reliably
+    const backendPageSize = maxResults === -1 ? pageSize : undefined;
+
+    const request = {
+      topicName: props.topic.topicName,
+      partitionId: partitionID,
+      startOffset,
+      startTimestamp: currentSearchParams?.startTimestamp ?? uiState.topicSettings.searchParams.startTimestamp,
+      maxResults,
+      pageSize: backendPageSize,
         filterInterpreterCode: encodeBase64(sanitizeString(filterCode)),
         includeRawPayload: true,
         keyDeserializer,
@@ -536,10 +606,11 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
         setFetchError(null);
         setSearchPhase('Searching...');
 
-        const messageSearch = createMessageSearch();
+        const search = createMessageSearch();
+      setMessageSearch(search);
         const startTime = Date.now();
 
-        const result = await messageSearch.startSearch(request, abortSignal).catch((err: Error) => {
+        const result = await search.startSearch(request, abortSignal).catch((err: Error) => {
           const msg = err.message ?? String(err);
           // biome-ignore lint/suspicious/noConsole: intentional console usage
           console.error(`error in searchTopicMessages: ${msg}`);
@@ -552,8 +623,8 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
         setMessages(result);
         setSearchPhase(null);
         setElapsedMs(endTime - startTime);
-        setBytesConsumed(messageSearch.bytesConsumed);
-        setTotalMessagesConsumed(messageSearch.totalMessagesConsumed);
+        setBytesConsumed(search.bytesConsumed);
+        setTotalMessagesConsumed(search.totalMessagesConsumed);
 
         return result;
       } catch (error: unknown) {
@@ -569,12 +640,12 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
       partitionID,
       startOffset,
       maxResults,
-      getSearchParams,
-      keyDeserializer,
-      valueDeserializer,
-      filters,
-    ]
-  );
+      pageSize,
+    getSearchParams,
+    keyDeserializer,
+    valueDeserializer,
+    filters,
+  ]);
 
   // Convert searchFunc to useCallback
   const searchFunc = useCallback(
@@ -628,6 +699,7 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
   );
 
   // Auto search when parameters change
+  // biome-ignore lint/correctness/useExhaustiveDependencies: forceRefresh is intentionally watched to trigger forced re-search
   useEffect(() => {
     // Set up auto-search with 100ms delay
     const timer = setTimeout(() => {
@@ -637,7 +709,71 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     appGlobal.searchMessagesFunc = searchFunc;
 
     return () => clearTimeout(timer);
-  }, [searchFunc]);
+  }, [searchFunc, forceRefresh]);
+
+  // Auto-load more messages when user reaches the last loaded page (pagination mode only)
+  useEffect(() => {
+    // Only auto-load in pagination mode (maxResults === -1) and when filters are not active
+    if (
+      maxResults !== -1 ||
+      filters.length > 0 ||
+      !messageSearch ||
+      !messageSearch.nextPageToken ||
+      isLoadingMore ||
+      searchPhase
+    ) {
+      return;
+    }
+
+    const totalLoadedPages = Math.ceil(messages.length / pageSize);
+    const isOnLastPage = pageIndex >= totalLoadedPages - 1;
+
+    if (isOnLastPage && messageSearch.nextPageToken) {
+      setIsLoadingMore(true);
+      messageSearch
+        .loadMore()
+        .then(() => {
+          setMessages([...messageSearch.messages]);
+        })
+        .catch((err) => {
+          toast({
+            title: 'Failed to load more messages',
+            description: (err as Error).message,
+            status: 'error',
+            duration: 5000,
+            isClosable: true,
+          });
+        })
+        .finally(() => {
+          setIsLoadingMore(false);
+        });
+    }
+  }, [
+    pageIndex,
+    maxResults,
+    filters.length,
+    messageSearch,
+    isLoadingMore,
+    searchPhase,
+    messages.length,
+    pageSize,
+    toast,
+  ]);
+
+  // Reset pagination when navigating back to page 1 in unlimited mode
+  // This prevents keeping many pages in memory and triggering excessive requests
+  useEffect(() => {
+    // Check if we're in unlimited mode and user navigated back to page 1 from a higher page
+    if (maxResults === -1 && pageIndex === 0 && prevPageIndexRef.current > 1 && messageSearch) {
+      // Clear the message search and state
+      setMessages([]);
+      setMessageSearch(null);
+      // Clear the search run ref and trigger a forced refresh
+      currentSearchRunRef.current = null;
+      setForceRefresh((prev) => prev + 1);
+    }
+    prevPageIndexRef.current = pageIndex;
+  }, [pageIndex, maxResults, messageSearch]);
 
   // Message Table rendering variables and functions
   const paginationParams = {
@@ -670,6 +806,7 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
     timestamp: {
       header: 'Timestamp',
       accessorKey: 'timestamp',
+      enableSorting: maxResults !== -1, // Disable sorting in unlimited pagination mode
       cell: ({
         row: {
           original: { timestamp },
@@ -696,6 +833,7 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
         ),
       size: hasKeyTags ? 300 : 1,
       accessorKey: 'key',
+      enableSorting: maxResults !== -1, // Disable sorting in unlimited pagination mode
       cell: ({ row: { original } }) => (
         <MessageKeyPreview
           msg={original}
@@ -723,6 +861,7 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
           'Value'
         ),
       accessorKey: 'value',
+      enableSorting: maxResults !== -1, // Disable sorting in unlimited pagination mode
       cell: ({ row: { original } }) => (
         <MessagePreview
           isCompactTopic={props.topic.cleanupPolicy.includes('compact')}
@@ -861,7 +1000,9 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
       label: (
         <Flex alignItems="center" gap={2}>
           <ReplyIcon />
-          <span data-testid="start-offset-newest">{`Newest - ${String(maxResults)}`}</span>
+          <span data-testid="start-offset-newest">
+            {maxResults === -1 ? 'Newest' : `Newest - ${String(maxResults)}`}
+          </span>
         </Flex>
       ),
     },
@@ -942,11 +1083,23 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
 
           <Label text="Max Results">
             <SingleSelect<number>
+              chakraStyles={maxResultsSelectChakraStyles}
               data-testid="max-results-select"
               onChange={(c) => {
                 setMaxResults(c);
               }}
-              options={[1, 3, 5, 10, 20, 50, 100, 200, 500].map((i) => ({ value: i }))}
+              options={[
+                // Only show "Unlimited" option when NOT on "Latest/Live" mode
+                ...(startOffset !== PartitionOffsetOrigin.End
+                  ? [
+                      {
+                        value: -1,
+                        label: '\u221e Unlimited',
+                      },
+                    ]
+                  : []),
+                ...[1, 3, 5, 10, 20, 50, 100, 200, 500].map((i) => ({ value: i })),
+              ]}
               value={maxResults}
             />
           </Label>
@@ -1020,10 +1173,10 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
           {Boolean(searchPhase && searchPhase.length > 0) && (
             <StatusIndicator
               bytesConsumed={prettyBytes(bytesConsumed)}
-              fillFactor={messages.length / maxResults}
+              fillFactor={maxResults === -1 ? 0 : messages.length / maxResults}
               identityKey="messageSearch"
               messagesConsumed={String(totalMessagesConsumed)}
-              progressText={`${messages.length} / ${maxResults}`}
+              progressText={maxResults === -1 ? `${messages.length}` : `${messages.length} / ${maxResults}`}
               // biome-ignore lint/style/noNonNullAssertion: not touching MobX observables
               statusText={searchPhase!}
             />
@@ -1197,6 +1350,11 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
               }
             )}
             onSortingChange={(newSorting) => {
+              // Don't allow sorting changes in unlimited pagination mode
+              if (maxResults === -1) {
+                return;
+              }
+
               const updatedSorting: SortingState = typeof newSorting === 'function' ? newSorting(sorting) : newSorting;
               setSortingState(updatedSorting);
             }}
@@ -1224,6 +1382,25 @@ export const TopicMessageView: FC<TopicMessageViewProps> = (props) => {
               />
             )}
           />
+
+          {/* Warning when filters are active with pagination */}
+          {maxResults === -1 && filters.length > 0 && messages.length > 0 && (
+            <Alert mt={4} status="info">
+              <AlertIcon />
+              <AlertDescription>
+                Auto-pagination is disabled when filters are active. Remove filters to enable automatic loading.
+              </AlertDescription>
+            </Alert>
+          )}
+
+          {/* Loading indicator when fetching more pages */}
+          {maxResults === -1 && isLoadingMore && (
+            <Flex justifyContent="center" mt={4}>
+              <Spinner mr={2} size="sm" />
+              <span>Loading more messages...</span>
+            </Flex>
+          )}
+
           <Button
             data-testid="save-messages-button"
             isDisabled={messages.length === 0}