Stream pipeline console output (#1004)

Co-authored-by: Tim Jacomb <[email protected]>
Co-authored-by: Tim Jacomb <[email protected]>

Author: das7pad

openapi.yaml

@@ -126,46 +126,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/ErrorResponse"
 
-  /consoleOutput:
-    get:
-      tags:
-        - Pipeline Overview
-      summary: Get console output for a step
-      description: |
-        Returns console output for a specific step or stage. Supports pagination
-        through startByte parameter.
-      operationId: getConsoleOutput
-      parameters:
-        - name: nodeId
-          in: query
-          required: true
-          description: The ID of the step/stage to get console output for
-          schema:
-            type: string
-        - name: startByte
-          in: query
-          required: false
-          description: |
-            Starting byte position. If negative, starts from end minus the absolute value.
-            If larger than output length, returns error. Default is -153600 (150KB from end).
-          schema:
-            type: integer
-            format: int64
-            default: -153600
-      responses:
-        "200":
-          description: Console output data
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ConsoleOutputResponse"
-        "400":
-          description: Missing or invalid parameters
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ErrorResponse"
-
   /exceptionText:
     get:
       tags:
@@ -543,33 +503,6 @@ components:
           type: boolean
           description: Whether the input step has parameters
 
-    ConsoleOutputResponse:
-      type: object
-      properties:
-        status:
-          type: string
-          enum: [ok]
-        data:
-          $ref: "#/components/schemas/ConsoleOutput"
-
-    ConsoleOutput:
-      type: object
-      properties:
-        text:
-          type: string
-          description: Console text content
-        startByte:
-          type: integer
-          format: int64
-          description: Starting byte position of this text chunk
-        endByte:
-          type: integer
-          format: int64
-          description: Ending byte position of this text chunk
-        nodeIsActive:
-          type: boolean
-          description: Whether the node is currently active/running
-
     NextBuildResponse:
       type: object
       properties:

pom.xml

@@ -33,8 +33,8 @@
     <changelist>999999-SNAPSHOT</changelist>
     <gitHubRepo>jenkinsci/pipeline-graph-view-plugin</gitHubRepo>
     <!-- Baseline Jenkins version you use to build the plugin. Users must have this version or newer to run. -->
-    <jenkins.baseline>2.516</jenkins.baseline>
-    <jenkins.version>2.532</jenkins.version>
+    <jenkins.baseline>2.528</jenkins.baseline>
+    <jenkins.version>2.534</jenkins.version>
     <node.version>24.10.0</node.version>
     <npm.version>11.6.2</npm.version>
     <spotless.check.skip>false</spotless.check.skip>
@@ -47,7 +47,7 @@
         <!-- Pick up common dependencies for the selected LTS line: https://github.com/jenkinsci/bom#usage -->
         <groupId>io.jenkins.tools.bom</groupId>
         <artifactId>bom-${jenkins.baseline}.x</artifactId>
-        <version>5601.v59f37270a_349</version>
+        <version>5622.vc9c3051619f5</version>
         <type>pom</type>
         <scope>import</scope>
       </dependency>

src/main/frontend/common/RestClient.tsx

@@ -47,6 +47,7 @@ export interface StepLogBufferInfo {
   endByte: number;
   pending?: Promise<void>;
   consoleAnnotator?: string;
+  hasTrailingNewLine?: boolean;
   lastFetched?: number;
   stopTailing?: boolean;
   exceptionText?: string[];
@@ -87,18 +88,23 @@ export async function getConsoleTextOffset(
   startByte: number,
   consoleAnnotator: string,
 ): Promise<ConsoleLogData | null> {
-  const headers = new Headers();
+  const headers = new Headers({ Accept: "multipart/form-data" });
   if (consoleAnnotator) headers.set("X-ConsoleAnnotator", consoleAnnotator);
   try {
     const response = await fetch(
-      `consoleOutput?nodeId=${stepId}&startByte=${startByte}`,
+      `../execution/node/${stepId}/log/logText/progressiveHtml?start=${startByte.toString()}`,
       { headers },
     );
     if (!response.ok) throw response.statusText;
-    const json = await response.json();
+    const data = await response.formData();
+    const text = data.get("text") as string;
+    const meta = JSON.parse(data.get("meta") as string);
     return {
-      ...json.data,
-      consoleAnnotator: response.headers.get("X-ConsoleAnnotator") || "",
+      text,
+      startByte: meta.start,
+      endByte: meta.end,
+      nodeIsActive: !meta.completed,
+      consoleAnnotator: meta.consoleAnnotator,
     };
   } catch (e) {
     console.error(`Caught error when fetching console: '${e}'`);

src/main/frontend/pipeline-console-view/pipeline-console/main/hooks/use-steps-poller.spec.ts

@@ -330,6 +330,7 @@ describe("incremental log fetching", function () {
           lines: ["0"],
           endByte: 2,
           consoleAnnotator: "0",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -348,6 +349,7 @@ describe("incremental log fetching", function () {
           lines: ["0", "1"],
           endByte: 4,
           consoleAnnotator: "1",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -366,6 +368,7 @@ describe("incremental log fetching", function () {
           lines: ["0", "1", "2", "3"],
           endByte: 8,
           consoleAnnotator: "3",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -384,6 +387,7 @@ describe("incremental log fetching", function () {
           lines: ["0", "1", "2", "3"],
           endByte: 8,
           consoleAnnotator: "empty",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -402,6 +406,66 @@ describe("incremental log fetching", function () {
           lines: ["0", "1", "2", "3", "4"],
           endByte: 10,
           consoleAnnotator: "4",
+          hasTrailingNewLine: true,
+        },
+      },
+    ],
+    "when gluing lines together": [
+      {
+        clickMoreStartByte: TAIL_CONSOLE_LOG,
+        fetchStartByte: TAIL_CONSOLE_LOG,
+        fetchConsoleAnnotator: "",
+        logData: {
+          text: "0\n1",
+          startByte: 0,
+          endByte: 3,
+          nodeIsActive: true,
+          consoleAnnotator: "1",
+        },
+        result: {
+          startByte: 0,
+          lines: ["0", "1"],
+          endByte: 3,
+          consoleAnnotator: "1",
+          hasTrailingNewLine: false,
+        },
+      },
+      {
+        clickMoreStartByte: TAIL_CONSOLE_LOG,
+        fetchStartByte: 3,
+        fetchConsoleAnnotator: "1",
+        logData: {
+          text: "2\n3",
+          startByte: 3,
+          endByte: 6,
+          nodeIsActive: true,
+          consoleAnnotator: "3",
+        },
+        result: {
+          startByte: 0,
+          lines: ["0", "12", "3"],
+          endByte: 6,
+          consoleAnnotator: "3",
+          hasTrailingNewLine: false,
+        },
+      },
+      {
+        clickMoreStartByte: TAIL_CONSOLE_LOG,
+        fetchStartByte: 6,
+        fetchConsoleAnnotator: "3",
+        logData: {
+          text: "\n4\n",
+          startByte: 6,
+          endByte: 9,
+          nodeIsActive: true,
+          consoleAnnotator: "4",
+        },
+        result: {
+          startByte: 0,
+          lines: ["0", "12", "3", "4"],
+          endByte: 9,
+          consoleAnnotator: "4",
+          hasTrailingNewLine: true,
         },
       },
     ],
@@ -422,6 +486,7 @@ describe("incremental log fetching", function () {
           lines: ["0"],
           endByte: 1_000_002,
           consoleAnnotator: "0",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -440,6 +505,7 @@ describe("incremental log fetching", function () {
           lines: ["0", "1"],
           endByte: 1_000_004,
           consoleAnnotator: "1",
+          hasTrailingNewLine: true,
         },
       },
     ],
@@ -460,6 +526,7 @@ describe("incremental log fetching", function () {
           lines: ["0"],
           endByte: 1_000_002,
           consoleAnnotator: "0",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -478,6 +545,7 @@ describe("incremental log fetching", function () {
           lines: ["xxx", "0"],
           endByte: 1_000_002,
           consoleAnnotator: "x",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -496,6 +564,7 @@ describe("incremental log fetching", function () {
           lines: ["yyy", "xxx", "0"],
           endByte: 1_000_002,
           consoleAnnotator: "y",
+          hasTrailingNewLine: true,
         },
       },
     ],
@@ -516,6 +585,7 @@ describe("incremental log fetching", function () {
           lines: ["0"],
           endByte: 1_000_002,
           consoleAnnotator: "0",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -534,6 +604,7 @@ describe("incremental log fetching", function () {
           lines: ["xxx", "0"],
           endByte: 1_000_002,
           consoleAnnotator: "x",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -552,6 +623,7 @@ describe("incremental log fetching", function () {
           lines: ["xxx", "0", "1"],
           endByte: 1_000_004,
           consoleAnnotator: "1",
+          hasTrailingNewLine: true,
         },
       },
       {
@@ -571,6 +643,7 @@ describe("incremental log fetching", function () {
           endByte: 1_000_004,
           consoleAnnotator: "y",
           stopTailing: true,
+          hasTrailingNewLine: true,
         },
       },
     ],

src/main/frontend/pipeline-console-view/pipeline-console/main/hooks/use-steps-poller.ts

@@ -62,14 +62,23 @@ async function updateStepBuffer(
   }
 
   const newLogLines = response.text.split("\n");
-  if (newLogLines[newLogLines.length - 1] === "") {
+  const hasTrailingNewLine = response.text.endsWith("\n");
+  if (!response.text || hasTrailingNewLine) {
     // Remove trailing empty new line caused by a) splitting an empty string or b) a trailing new line character in the response.
     newLogLines.pop();
   }
 
   const exceptionText = stepBuffer.exceptionText || [];
   if (stepBuffer.endByte > 0 && stepBuffer.endByte === startByte) {
     stepBuffer.lines.length -= exceptionText.length;
+    if (
+      !stepBuffer.hasTrailingNewLine &&
+      stepBuffer.lines.length > 0 &&
+      newLogLines.length > 0
+    ) {
+      // Combine a previously broken up line back together.
+      stepBuffer.lines[stepBuffer.lines.length - 1] += newLogLines.shift();
+    }
     stepBuffer.lines = [...stepBuffer.lines, ...newLogLines, ...exceptionText];
   } else {
     stepBuffer.lines = newLogLines.concat(exceptionText);
@@ -78,6 +87,10 @@ async function updateStepBuffer(
 
   stepBuffer.endByte = response.endByte;
   stepBuffer.consoleAnnotator = response.consoleAnnotator;
+  if (response.text) {
+    // Only overwrite when more text was available.
+    stepBuffer.hasTrailingNewLine = hasTrailingNewLine;
+  }
   if (!response.nodeIsActive) {
     // We've reached the end of the log now.
     stepBuffer.stopTailing = true;