Update the estimated number of rows (#22)

* update the number of rows on new estimate

* remove unneeded check

Author: severo

package-lock.json

@@ -16,7 +16,7 @@
   },
   "dependencies": {
     "cosovo": "0.2.0",
-    "hightable": "0.22.2",
+    "hightable": "0.24.1",
     "react": "19.2.0",
     "react-dom": "19.2.0"
   },

package.json

@@ -1,5 +1,5 @@
 import { type DataFrame, HighTable } from 'hightable'
-import { type ReactNode } from 'react'
+import { type ReactNode, useEffect, useState } from 'react'
 
 import Loading from './Loading.js'
 
@@ -28,6 +28,23 @@ export default function Page({
   setError,
   iframe = false,
 }: PageProps): ReactNode {
+  const [numRowsEstimate, setNumRowsEstimate] = useState<{ numRows: number, isEstimate: boolean } | undefined>(undefined)
+  useEffect(() => {
+    if (!df) return
+    /**
+     * Handle num rows update event
+     */
+    function onNumRowsUpdate() {
+      if (!df) return
+      setNumRowsEstimate({ numRows: df.numRows, isEstimate: df.metadata?.isNumRowsEstimated ?? false })
+    }
+    onNumRowsUpdate()
+    df.eventTarget?.addEventListener('numrowschange', onNumRowsUpdate)
+    return () => {
+      df.eventTarget?.removeEventListener('numrowschange', onNumRowsUpdate)
+    }
+  }, [df])
+
   return (
     <>
       {iframe ? '' : <div className="top-header">{name}</div>}
@@ -37,14 +54,14 @@ export default function Page({
             {formatFileSize(byteLength)}
           </span>
         )}
-        {df
+        {numRowsEstimate
           ? (
               <span>
-                {df.numRows.toLocaleString()}
+                {numRowsEstimate.numRows.toLocaleString()}
                 {' '}
                 row
-                {df.numRows > 1 ? 's' : ''}
-                {df.metadata?.isNumRowsEstimated ? ' (estimated)' : ''}
+                {numRowsEstimate.numRows > 1 ? 's' : ''}
+                {numRowsEstimate.isEstimate ? ' (estimated)' : ''}
               </span>
             )
           : null}

src/Page.tsx

@@ -1,7 +1,15 @@
 import type { Newline, ParseResult } from 'cosovo'
+import { createEventTarget } from 'hightable'
 
 import { checkNonNegativeInteger } from './helpers.js'
 
+export interface CSVCacheEvents {
+  'num-rows-estimate-updated': {
+    numRows: number
+    isEstimate: boolean
+  }
+}
+
 /**
  * A byte range in a CSV file, with the parsed rows
  */
@@ -192,6 +200,14 @@ export class CSVCache {
    * The average number of bytes per row, used for estimating row positions
    */
   #averageRowByteCount: number | undefined = undefined
+  /**
+   * The estimated number of rows in the CSV file
+   */
+  #numRowsEstimate: { numRows: number, isEstimate: boolean } = { numRows: 0, isEstimate: true }
+  /**
+   * An event target to emit events
+   */
+  #eventTarget = createEventTarget<CSVCacheEvents>()
 
   constructor({ columnNames, headerByteCount, byteLength, delimiter, newline }: { columnNames: string[], headerByteCount?: number, byteLength: number, delimiter: string, newline: Newline }) {
     headerByteCount ??= 0
@@ -267,6 +283,22 @@ export class CSVCache {
     return this.#newline
   }
 
+  /**
+   * Get an estimate of the total number of rows in the CSV file
+   * @returns The estimated number of rows and if it's an estimate
+   */
+  get numRowsEstimate(): { numRows: number, isEstimate: boolean } {
+    return this.#numRowsEstimate
+  }
+
+  /**
+   * Get the event target to listen to cache events
+   * @returns The event target
+   */
+  get eventTarget(): ReturnType<typeof createEventTarget<CSVCacheEvents>> {
+    return this.#eventTarget
+  }
+
   /**
    * Update the average row byte count based on the cached rows
    */
@@ -275,10 +307,11 @@ export class CSVCache {
     const rowCount = this.#serial.rowCount + this.#random.reduce((sum, range) => sum + range.rowCount, 0)
     if (rowCount === 0) {
       this.#averageRowByteCount = undefined
-      return
     }
-    const averageRowByteCount = rowByteCount / rowCount
-    this.#averageRowByteCount = averageRowByteCount
+    else {
+      this.#averageRowByteCount = rowByteCount / rowCount
+    }
+    this.#updateNumRowsEstimate()
   }
 
   /**
@@ -322,19 +355,19 @@ export class CSVCache {
   }
 
   /**
-   * Get an estimate of the total number of rows in the CSV file
-   * @returns The estimated number of rows and if it's an estimate
+   * Update the estimated number of rows in the CSV file
    */
-  get numRowsEstimate(): { numRows: number, isEstimate: boolean } {
+  #updateNumRowsEstimate(): void {
     const averageRowByteCount = this.averageRowByteCount
     const numRows = this.allRowsCached
       ? this.rowCount
       : averageRowByteCount === 0 || averageRowByteCount === undefined
         ? 0
         : Math.round((this.#byteLength - this.headerByteCount) / averageRowByteCount)
-    return {
-      numRows,
-      isEstimate: !this.allRowsCached,
+    const isEstimate = !this.allRowsCached
+    if (this.#numRowsEstimate.numRows !== numRows || this.#numRowsEstimate.isEstimate !== isEstimate) {
+      this.#numRowsEstimate = { numRows, isEstimate }
+      this.#eventTarget.dispatchEvent(new CustomEvent('num-rows-estimate-updated'))
     }
   }
 

src/cache.ts

@@ -27,6 +27,7 @@ interface Params {
   initialRowCount?: number // number of rows to fetch at dataframe creation
 }
 
+// Note that when sending a 'numrowsupdate' event, the isNumRowsEstimated flag is also updated if needed
 export type CSVDataFrame = DataFrame<{ isNumRowsEstimated: boolean }>
 
 /**
@@ -45,10 +46,12 @@ export async function csvDataFrame(params: Params): Promise<CSVDataFrame> {
 
   const eventTarget = createEventTarget<DataFrameEvents>()
   const cache = await initializeCSVCachefromURL({ url, byteLength, chunkSize, initialRowCount })
-  const { numRows, isEstimate } = cache.numRowsEstimate
-  const metadata = {
-    isNumRowsEstimated: isEstimate,
-  }
+  cache.eventTarget.addEventListener('num-rows-estimate-updated', () => {
+    // propagate event
+    eventTarget.dispatchEvent(new CustomEvent('numrowschange'))
+  })
+  // no need to remove the listener, as the cache has the same lifetime as the dataframe
+
   const columnDescriptors: DataFrame['columnDescriptors'] = cache.columnNames.map(name => ({ name }))
 
   /**
@@ -141,24 +144,21 @@ export async function csvDataFrame(params: Params): Promise<CSVDataFrame> {
   }): Promise<void> {
     checkSignal(signal)
 
+    // until the CSV is fully loaded, we don't know the exact number of rows
+    const numRows = cache.allRowsCached ? cache.rowCount : Infinity
     validateFetchParams({
       rowStart,
       rowEnd,
       columns,
       orderBy,
       data: {
-        numRows: Infinity, // we don't (always) know the exact number of rows yet
+        numRows,
         columnDescriptors,
       },
     })
 
     if (cache.allRowsCached) {
       // all rows are already cached
-      if (rowEnd > cache.rowCount) {
-        // requested rows are beyond the end of the file
-        throw new Error(`Requested rows are beyond the end of the file: ${rowEnd} > ${cache.rowCount}`)
-      }
-      // else nothing to do
       return
     }
 
@@ -262,8 +262,14 @@ export async function csvDataFrame(params: Params): Promise<CSVDataFrame> {
   }
 
   return {
-    metadata,
-    numRows,
+    metadata: {
+      get isNumRowsEstimated() {
+        return cache.numRowsEstimate.isEstimate
+      },
+    },
+    get numRows() {
+      return cache.numRowsEstimate.numRows
+    },
     columnDescriptors,
     getCell,
     getRowNumber,