only store the first 50 rows initially + the whole chunk

Author: severo

src/dataframe.ts

@@ -17,7 +17,7 @@ import { CSVCache } from './cache'
 import { checkNonNegativeInteger } from './helpers.js'
 
 const defaultChunkSize = 100 * 1024 // 100 KB
-const defaultInitialRowCount = 500
+const defaultInitialRowCount = 50
 // const paddingRowCount = 20 // fetch a bit before and after the requested range, to avoid cutting rows
 
 interface Params {
@@ -215,6 +215,7 @@ export async function csvDataFrame(params: Params): Promise<CSVDataFrame> {
 
         // Store the new row in the cache
         const isEmpty = isEmptyLine(result.row)
+        // v8 ignore else -- @preserve
         if (!cache.isStored({ byteOffset: result.meta.byteOffset })) {
           cache.store({
             cells: isEmpty ? undefined : result.row,
@@ -227,6 +228,10 @@ export async function csvDataFrame(params: Params): Promise<CSVDataFrame> {
             eventTarget.dispatchEvent(new CustomEvent('resolve'))
           }
         }
+        else {
+          // the row should not be already stored, but double check
+          console.debug('Row already stored, should not happen, skipping', { byteOffset: result.meta.byteOffset, row })
+        }
         if (!isEmpty) {
           row++
         }
@@ -295,7 +300,7 @@ async function initializeCSVCachefromURL({ url, byteLength, chunkSize, initialRo
       cache = CSVCache.fromHeader({ header: result, byteLength })
       continue
     }
-    else if (cache.rowCount >= initialRowCount) {
+    else if (cache.rowCount >= initialRowCount && result.meta.byteOffset > 0.9 * chunkSize) {
       // enough rows for now
       break
     }

test/dataframe.test.ts

@@ -50,12 +50,27 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 1, column: 'b' })).toStrictEqual({ value: '5' })
       expect(df.getCell({ row: 2, column: 'b' })).toBeUndefined()
       revoke()
     })
 
+    it('should fetch more than initial rows when specified if the chunk size is bigger', async () => {
+      const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n'
+      const { url, revoke, fileSize } = toURL(text, { withNodeWorkaround: true })
+      const df = await csvDataFrame({
+        url,
+        byteLength: fileSize,
+        initialRowCount: 2,
+        chunkSize: 500,
+      })
+      expect(df.getCell({ row: 1, column: 'b' })).toStrictEqual({ value: '5' })
+      expect(df.getCell({ row: 2, column: 'b' })).not.toBeUndefined()
+      revoke()
+    })
+
     it.each([
       { text: 'a,b,c\n1111,2222,3333\nn44,55,66\n77,88,99\n', expectedRows: 2 },
       { text: 'a,b,c\n11,22,33\n44,55,66\n77,88,99\n', expectedRows: 3 },
@@ -66,6 +81,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       // with only one row loaded, the average row size is not accurate enough to estimate the number of rows
       expect(df.numRows).toBe(expectedRows) // the estimate is not perfect
@@ -80,6 +96,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 0,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 1, column: 'b' })).toBeUndefined()
       revoke()
@@ -170,6 +187,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined()
       revoke()
@@ -182,6 +200,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 5, column: 'a' })).toBeUndefined()
       revoke()
@@ -242,6 +261,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getRowNumber({ row: 2 })).toBeUndefined()
       revoke()
@@ -254,6 +274,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getRowNumber({ row: 5 })).toBeUndefined()
       revoke()
@@ -290,6 +311,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 2, rowEnd: 5 })
@@ -306,6 +328,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 1, rowEnd: 10 })
@@ -366,6 +389,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 2, rowEnd: 10 })
@@ -380,6 +404,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 3, rowEnd: 4 })
@@ -396,6 +421,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 2, rowEnd: 5 })
@@ -412,6 +438,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined()
       await df.fetch?.({ rowStart: 3, rowEnd: 5 })
@@ -455,6 +482,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '111111' })
       expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined()
@@ -484,6 +512,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '111111' })
       expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined()
@@ -514,6 +543,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 2,
+        chunkSize: 5,
       })
 
       let resolveEventCount = 0
@@ -541,6 +571,7 @@ describe('csvDataFrame', () => {
         url,
         byteLength: fileSize,
         initialRowCount: 1,
+        chunkSize: 5,
       })
       expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '1' })
       expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined()