diff --git a/package-lock.json b/package-lock.json index 3567248..7cdd1fb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "@vitejs/plugin-react": "5.1.2", "@vitest/coverage-v8": "4.0.15", "@vitest/ui": "4.0.15", + "baseline-browser-mapping": "2.9.5", "eslint": "9.39.1", "eslint-plugin-jsdoc": "61.5.0", "eslint-plugin-react-dom": "2.3.13", @@ -2224,9 +2225,9 @@ "license": "MIT" }, "node_modules/baseline-browser-mapping": { - "version": "2.8.12", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.12.tgz", - "integrity": "sha512-vAPMQdnyKCBtkmQA6FMCBvU9qFIppS3nzyXnEM+Lo2IAhG4Mpjv9cCxMudhgV3YdNNJv6TNqXy97dfRVL2LmaQ==", + "version": "2.9.5", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.5.tgz", + "integrity": "sha512-D5vIoztZOq1XM54LUdttJVc96ggEsIfju2JBvht06pSzpckp3C7HReun67Bghzrtdsq9XdMGbSSB3v3GhMNmAA==", "dev": true, "license": "Apache-2.0", "bin": { diff --git a/package.json b/package.json index 7abf50f..6b48b80 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@vitejs/plugin-react": "5.1.2", "@vitest/coverage-v8": "4.0.15", "@vitest/ui": "4.0.15", + "baseline-browser-mapping": "2.9.5", "eslint": "9.39.1", "eslint-plugin-jsdoc": "61.5.0", "eslint-plugin-react-dom": "2.3.13", diff --git a/src/cache.ts b/src/cache.ts index 060812d..2c9328b 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,12 +1,43 @@ import type { Newline, ParseResult } from 'cosovo' -import { createEventTarget } from 'hightable' -import { checkNonNegativeInteger } from './helpers.js' +import { checkInteger, checkNonNegativeInteger } from './helpers.js' -export interface CSVCacheEvents { - 'num-rows-estimate-updated': { - numRows: number - isEstimate: boolean +/** + * Cache of parsed rows + */ +export class RowsCache { + #rows: string[][] = [] + #byteCount = 0 + + get rows(): string[][] { + return this.#rows + } + + get byteCount(): number { + return this.#byteCount + } + + get numRows(): number { + return this.#rows.length + } + + append(row: { byteCount: number, cells: string[] }) { + checkNonNegativeInteger(row.byteCount) + this.#rows.push(row.cells.slice()) + this.#byteCount += row.byteCount + } + + prepend(row: { byteCount: number, cells: string[] }) { + checkNonNegativeInteger(row.byteCount) + this.#rows.unshift(row.cells.slice()) + this.#byteCount += row.byteCount + } + + merge(other: RowsCache) { + for (const row of other.rows) { + this.#rows.push(row) + } + this.#byteCount += other.byteCount } } @@ -15,22 +46,20 @@ export interface CSVCacheEvents { */ export class CSVRange { #firstByte: number // byte position of the start of the range (excludes the ignored bytes if the range starts in the middle of a row) - #byteCount = 0 // number of bytes in the range - #rowByteCount = 0 // number of bytes in the range's rows (excludes ignored bytes) - #rows: string[][] = [] // sorted array of the range rows, filtering out the empty rows and the header if any - #firstRow: number // index of the first row in the range (0-based) + #byteCount: number = 0 // number of bytes in the range (includes ignored bytes) + #rowsCache: RowsCache // the cached rows. It excludes the ignored rows (empty rows and header if any) - constructor({ firstByte, firstRow }: { firstByte: number, firstRow: number }) { + constructor({ firstByte }: { firstByte: number }) { this.#firstByte = checkNonNegativeInteger(firstByte) - this.#firstRow = checkNonNegativeInteger(firstRow) + this.#rowsCache = new RowsCache() } /* - * Get the number of rows in the range - * @returns The number of rows in the range + * Get the slice of cached rows + * @returns The slice of cached rows */ - get rowCount() { - return this.#rows.length + get rowsCache() { + return this.#rowsCache } /** @@ -50,46 +79,11 @@ export class CSVRange { } /** - * Get the first row number of the range - * @returns The first row number of the range - */ - get firstRow(): number { - return this.#firstRow - } - - /** - * Set the first row number of the range - * @param value The first row number + * Get the first byte of the next row in the range + * @returns The next row's first byte */ - set firstRow(value: number) { - this.#firstRow = checkNonNegativeInteger(value) - } - - /** - * Get the row number and first byte of the next row in the range - * @returns The next row number and first byte - */ - get next(): { row: number, firstByte: number } { - return { - row: this.#firstRow + this.#rows.length, - firstByte: this.#firstByte + this.#byteCount, - } - } - - /** - * Get the rows in the range - * @returns The rows in the range - */ - get rows(): string[][] { - return this.#rows - } - - /** - * Get the number of bytes covered by the rows in the range - * @returns The number of bytes in the range's rows - */ - get rowByteCount(): number { - return this.#rowByteCount + get nextByte(): number { + return this.#firstByte + this.#byteCount } /** @@ -107,8 +101,7 @@ export class CSVRange { } this.#byteCount = row.byteOffset + row.byteCount - this.#firstByte if (row.cells) { - this.#rows.push(row.cells.slice()) - this.#rowByteCount += row.byteCount + this.#rowsCache.append({ cells: row.cells, byteCount: row.byteCount }) } } @@ -128,25 +121,8 @@ export class CSVRange { this.#firstByte = row.byteOffset this.#byteCount += row.byteCount if (row.cells) { - this.#firstRow -= 1 - this.#rows.unshift(row.cells.slice()) - this.#rowByteCount += row.byteCount - } - } - - /** - * Get the cells of a given row - * @param options Options - * @param options.row The row number (0-based) - * @returns The cells of the row, or undefined if the row is not in this range - */ - getCells({ row }: { row: number }): string[] | undefined { - checkNonNegativeInteger(row) - const rowIndex = row - this.#firstRow - if (rowIndex < 0 || rowIndex >= this.#rows.length) { - return undefined + this.#rowsCache.prepend({ cells: row.cells, byteCount: row.byteCount }) } - return this.#rows[rowIndex] } /** @@ -154,14 +130,22 @@ export class CSVRange { * @param followingRange The range to merge */ merge(followingRange: CSVRange): void { - if (this.next.firstByte !== followingRange.firstByte) { + if (this.nextByte !== followingRange.firstByte) { throw new Error('Cannot merge ranges: not contiguous') } this.#byteCount += followingRange.byteCount - this.#rowByteCount += followingRange.rowByteCount - for (const row of followingRange.rows) { - this.#rows.push(row) - } + this.#rowsCache.merge(followingRange.rowsCache) + } + + /** + * Get the cells of a given cached row (0-based index in this range, no ignored rows) + * @param rowIndex The row index in this range (0-based) + * @returns The cells of the row, or undefined if the row is not in this range + */ + getRow(rowIndex: number): string[] | undefined { + // if negative or out of bounds, return undefined + checkInteger(rowIndex) + return this.#rowsCache.rows[rowIndex] } } @@ -197,18 +181,6 @@ export class CSVCache { * The CSV newline character(s) */ #newline: Newline - /** - * The average number of bytes per row, used for estimating row positions - */ - #averageRowByteCount: number | undefined = undefined - /** - * The estimated number of rows in the CSV file - */ - #numRowsEstimate: { numRows: number, isEstimate: boolean } = { numRows: 0, isEstimate: true } - /** - * An event target to emit events - */ - #eventTarget = createEventTarget() constructor({ columnNames, headerByteCount, byteLength, delimiter, newline }: { columnNames: string[], headerByteCount?: number, byteLength: number, delimiter: string, newline: Newline }) { headerByteCount ??= 0 @@ -225,15 +197,40 @@ export class CSVCache { this.#headerByteCount = headerByteCount this.#delimiter = delimiter this.#newline = newline - this.#serial = new CSVRange({ firstByte: 0, firstRow: 0 }) + + const serial = new CSVRange({ firstByte: 0 }) // Account for the header row and previous ignored rows if any - this.#serial.append({ + serial.append({ byteOffset: 0, byteCount: headerByteCount, }) + this.#serial = serial this.#random = [] + } - this.#updateAverageRowByteCount() + /** + * Create a CSVCache from a header row + * @param options Options + * @param options.header The parsed header row + * @param options.byteLength The byte length of the CSV file + * @returns A new CSVCache instance + */ + static fromHeader({ header, byteLength }: { header: ParseResult, byteLength: number }): CSVCache { + return new CSVCache({ + columnNames: header.row, + byteLength, + delimiter: header.meta.delimiter, + newline: header.meta.newline, + headerByteCount: header.meta.byteOffset + header.meta.byteCount, + }) + } + + /** + * Get the number of bytes in the CSV file + * @returns The number of bytes in the CSV file + */ + get byteLength(): number { + return this.#byteLength } /** @@ -252,14 +249,6 @@ export class CSVCache { return this.#headerByteCount } - /** - * Get the number of rows in the cache - * @returns The number of rows in the cache - */ - get rowCount(): number { - return this.#serial.rowCount + this.#random.reduce((sum, range) => sum + range.rowCount, 0) - } - /** * Get the CSV delimiter * @returns The CSV delimiter @@ -268,14 +257,6 @@ export class CSVCache { return this.#delimiter } - /** - * Get the number of columns - * @returns The number of columns - */ - get columnCount(): number { - return this.#columnNames.length - } - /** * Get the CSV newline character(s) * @returns The CSV newline character(s) @@ -285,91 +266,196 @@ export class CSVCache { } /** - * Get an estimate of the total number of rows in the CSV file - * @returns The estimated number of rows and if it's an estimate + * Get the serial range + * @returns The serial range */ - get numRowsEstimate(): { numRows: number, isEstimate: boolean } { - return this.#numRowsEstimate + get serialRange(): CSVRange { + return this.#serial } /** - * Get the event target to listen to cache events - * @returns The event target + * Get the random access ranges + * @returns The random access ranges */ - get eventTarget(): ReturnType> { - return this.#eventTarget + get randomRanges(): CSVRange[] { + return this.#random.slice() } /** - * Update the average row byte count based on the cached rows + * Check if the entire CSV file is cached + * @returns True if the entire CSV file is cached */ - #updateAverageRowByteCount(): void { - const rowByteCount = this.#serial.rowByteCount + this.#random.reduce((sum, range) => sum + range.rowByteCount, 0) - const rowCount = this.#serial.rowCount + this.#random.reduce((sum, range) => sum + range.rowCount, 0) - if (rowCount === 0) { - this.#averageRowByteCount = undefined + get complete(): boolean { + // v8 ignore if -- @preserve + if (this.#serial.nextByte > this.#byteLength) { + throw new Error('Inconsistent state: serial range exceeds the file length') } - else { - this.#averageRowByteCount = rowByteCount / rowCount + const complete = this.#serial.nextByte === this.#byteLength + // v8 ignore if -- @preserve + if (complete && this.#random.length > 0) { + throw new Error('Inconsistent state: serial range covers the entire file, but there are random ranges') } - this.#updateNumRowsEstimate() + return complete } /** - * Re-assign row numbers in random ranges to reduce overlaps + * Store a new row + * If the byte range is already cached, false is immediately returned. + * If only part of the byte range is cached, an error is thrown. + * @param row The row to store. + * @param row.byteOffset The byte offset of the row in the file. + * @param row.byteCount The number of bytes of the row. + * @param row.cells The cells of the row. If not provided, the row is considered ignored and not cached (i.e., empty or header). + * @returns True if the row was stored successfully. */ - updateRowEstimates(): void { - const averageRowByteCount = this.averageRowByteCount - if (averageRowByteCount === undefined || averageRowByteCount === 0) { - return + store(row: { byteOffset: number, byteCount: number, cells?: string[] }): boolean { + // TODO(SL): forbid storing rows before the headerByteCount? + checkNonNegativeInteger(row.byteOffset) + checkNonNegativeInteger(row.byteCount) + + if (row.byteOffset + row.byteCount > this.#byteLength) { + throw new Error('Cannot store the row: byte range is out of bounds') } - let previousRange = this.#serial + let leftRange = this.#serial + + // loop on the ranges to find where to put the row + for (const [i, rightRange] of [...this.#random, undefined].entries()) { + // at this point, the row cannot start before the left range - // loop on the random ranges - for (const range of this.#random) { // v8 ignore if -- @preserve - if (range.firstByte <= previousRange.next.firstByte) { - // should not happen - throw new Error('Cannot update row estimates: overlap with previous range') + if (row.byteOffset < leftRange.firstByte) { + throw new Error('Inconsistent state: row is before the left range') + } + + if (row.byteOffset < leftRange.nextByte) { + // the row starts inside the left range + if (row.byteOffset + row.byteCount > leftRange.nextByte) { + // but it ends after the left range: we cannot store the row + throw new Error('Cannot store the row: the first bytes are already cached, but not the last ones.') + } + // it ends inside the left range: the row is already cached + return false + } + // at this point, the row starts after the left range + + if (rightRange && row.byteOffset >= rightRange.firstByte) { + // the row starts inside the right range or after. Go to the next range. + leftRange = rightRange + continue + } + + // at this point, the row starts before the right range (or there is no right range) + + if (rightRange && row.byteOffset + row.byteCount > rightRange.firstByte) { + // but it ends after the start of the right range: we cannot store the row + throw new Error('Cannot store the row: the first bytes are not cached, but other bytes are already cached.') + } + + // at this point, the row ends before the right range (or there is no right range). + // The row can be contiguous to the left range, or to the right range, or to both, or isolated. + // It will be stored. + + if (row.byteOffset === leftRange.nextByte) { + // The row is contiguous to the left range: append it. + leftRange.append(row) + // merge with the right range if needed (after appending to the left range) + if (rightRange && leftRange.nextByte === rightRange.firstByte) { + // merge the left and right ranges + this.#merge(leftRange, rightRange) + } + return true } - const firstRow = Math.max( - // ensure at least one row gap - previousRange.next.row + 1, - // estimate based on byte position - Math.round(previousRange.next.row + (range.firstByte - previousRange.next.firstByte) / averageRowByteCount), - ) + // at this point, the row is not contiguous to the left range + + if (rightRange && row.byteOffset + row.byteCount === rightRange.firstByte) { + // The row is contiguous to the right range: prepend it. + rightRange.prepend(row) + return true + } - range.firstRow = firstRow + // at this point, the row is not contiguous to either range - previousRange = range + // create a new random range and insert it after the left range + const newRange = new CSVRange({ firstByte: row.byteOffset }) + newRange.append(row) + this.#random.splice(i, 0, newRange) + return true } + + // v8 ignore next -- @preserve + throw new Error('Inconsistent state: this point should not be reachable') } - get averageRowByteCount(): number | undefined { - return this.#averageRowByteCount + /** + * Merge two CSV ranges + * @param range The first range. It can be the serial range, or a random range. + * @param followingRange The second range, must be immediately after the first range. It is a random range. + */ + #merge(range: CSVRange, followingRange: CSVRange): void { + const index = this.#random.indexOf(followingRange) + // v8 ignore if -- @preserve + if (index === -1) { + throw new Error('Cannot merge ranges: following range not found in cache') + } + range.merge(followingRange) + // remove followingRange from the random ranges + this.#random.splice(index, 1) } +} + +export class Estimator { + /** + * The CSV cache + */ + #cache: CSVCache + + /** + * The average number of bytes per row, used for estimating row positions. Undefined if the cache is complete. + */ + #averageRowByteCount: number | undefined = 0 - get allRowsCached(): boolean { - return this.#serial.next.firstByte >= this.#byteLength + constructor({ cache }: { cache: CSVCache }) { + this.#cache = cache } /** - * Update the estimated number of rows in the CSV file + * Get a copy of the estimator + * @returns A copy of the estimator */ - #updateNumRowsEstimate(): void { - const averageRowByteCount = this.averageRowByteCount - const numRows = this.allRowsCached - ? this.rowCount - : averageRowByteCount === 0 || averageRowByteCount === undefined - ? 0 - : Math.round((this.#byteLength - this.headerByteCount) / averageRowByteCount) - const isEstimate = !this.allRowsCached - if (this.#numRowsEstimate.numRows !== numRows || this.#numRowsEstimate.isEstimate !== isEstimate) { - this.#numRowsEstimate = { numRows, isEstimate } - this.#eventTarget.dispatchEvent(new CustomEvent('num-rows-estimate-updated')) - } + copy(): Estimator { + const copy = new Estimator({ cache: this.#cache }) + copy.#averageRowByteCount = this.#averageRowByteCount + return copy + } + + /** + * Get the estimated number of rows in the CSV file + * @returns The estimated number of rows + */ + get numRows(): number { + return this.#averageRowByteCount === 0 + ? 0 + : this.#averageRowByteCount === undefined + ? this.#computeNumCachedRows() + : Math.round((this.#cache.byteLength - this.#cache.headerByteCount) / this.#averageRowByteCount) + } + + /** + * Get if the number of rows is estimated + * @returns True if the number of rows is estimated + */ + get isNumRowsEstimated(): boolean { + return this.#averageRowByteCount !== undefined + } + + /** + * Get the maximum number of rows (infinity if estimated, numRows if exact) + * @returns The maximum number of rows + */ + get maxNumRows(): number { + return this.isNumRowsEstimated ? Infinity : this.numRows } /** @@ -378,19 +464,9 @@ export class CSVCache { * @param options.row The row number (0-based) * @returns The cells of the row, or undefined if the row is not in this range */ - #getCells({ row }: { row: number }): string[] | undefined { - const cells = this.#serial.getCells({ row }) - if (cells !== undefined) { - return cells - } - // find the range containing this row - for (const range of this.#random) { - const cells = range.getCells({ row }) - if (cells !== undefined) { - return cells - } - } - return undefined + isStored({ row }: { row: number }): boolean { + const cells = this.#getCells({ row }) + return cells !== undefined } /** @@ -402,7 +478,7 @@ export class CSVCache { */ getCell({ row, column }: { row: number, column: number }): { value: string } | undefined { checkNonNegativeInteger(column) - if (column >= this.columnCount) { + if (column >= this.#cache.columnNames.length) { throw new Error(`Column index out of bounds: ${column}`) } const cells = this.#getCells({ row }) @@ -422,182 +498,135 @@ export class CSVCache { * @returns The row number, or undefined if not found */ getRowNumber({ row }: { row: number }): { value: number } | undefined { - if (this.#getCells({ row }) === undefined) { + if (row >= 0 && row < this.numRows) { + return { value: row } + } + } + + // TODO(SL): look at the ranges to improve the estimation, in particular to avoid gaps between successive rows + // TODO(SL): also tell if it's a guess or exact + guessByteOffset({ row }: { row: number }): number | undefined { + // special case: even if averageRowByteCount is undefined or 0, we know the byte offset of row 0 + if (row === 0) { + return this.#cache.headerByteCount + } + if (this.#averageRowByteCount === 0 || this.#averageRowByteCount === undefined) { return undefined } - return { value: row } + return Math.max(0, + Math.min(this.#cache.byteLength - 1, + this.#cache.headerByteCount + Math.round(row * this.#averageRowByteCount), + ), + ) } /** - * Store a new row - * @param row The row to store. - * @param row.byteOffset The byte offset of the row in the file. - * @param row.byteCount The number of bytes of the row. - * @param row.firstRow The first row index (0-based) used if a new range is created. - * @param row.cells The cells of the row. If not provided, the row is considered ignored and not cached (i.e., empty or header). + * Refresh the internal state (average row byte count) + * Don't update the internal state if the change is not significant (<1%) + * @returns True if the internal state has been updated */ - store(row: { byteOffset: number, byteCount: number, firstRow: number, cells?: string[] }): void { - checkNonNegativeInteger(row.byteOffset) - checkNonNegativeInteger(row.byteCount) - if (row.byteOffset + row.byteCount > this.#byteLength) { - throw new Error('Cannot store the row: byte range is out of bounds') - } - - let previousRange = this.#serial - - // loop on the ranges to find where to put the row - for (const [i, nextRange] of [...this.#random, undefined].entries()) { - if (row.byteOffset < previousRange.next.firstByte) { - throw new Error('Cannot store the row: overlap with previous range') - } - - // the row is after the next range - if (nextRange && row.byteOffset >= nextRange.next.firstByte) { - previousRange = nextRange - continue - } - - if (nextRange && row.byteOffset + row.byteCount > nextRange.firstByte) { - throw new Error('Cannot store the row: overlap with next range') - } + refresh(): boolean { + const numCachedBytes = this.#computeNumCachedBytes() + const numCachedRows = this.#computeNumCachedRows() + const complete = this.#cache.complete - // append to the previous range - if (row.byteOffset === previousRange.next.firstByte) { - previousRange.append(row) - // merge with the next range if needed - if (nextRange && previousRange.next.firstByte === nextRange.firstByte) { - // merge nextRange into previousRange - this.#merge(previousRange, nextRange) - } - break + if (complete) { + if (this.#averageRowByteCount === undefined) { + // already in the same state + return false } + // update + this.#averageRowByteCount = undefined + return true + } - // prepend to the next range - if (nextRange && row.byteOffset + row.byteCount === nextRange.firstByte) { - nextRange.prepend(row) - break - } + // v8 ignore if -- @preserve + if (this.#averageRowByteCount === undefined) { + throw new Error('Incoherent state: the cache state cannot go from complete to incomplete') + } - // create a new random range between previousRange and nextRange - // Note that we might have a situation where firstRow overlaps with other ranges. - const newRange = new CSVRange({ firstByte: row.byteOffset, firstRow: row.firstRow }) - newRange.append(row) - this.#random.splice(i, 0, newRange) - break + if (numCachedRows === 0) { + // no progress + return false } - // Update the average row byte count - this.#updateAverageRowByteCount() - } + const averageRowByteCount = numCachedBytes / numCachedRows - /** - * Merge two CSV ranges - * @param range The first range. It can be the serial range, or a random range. - * @param followingRange The second range, must be immediately after the first range. It is a random range. - */ - #merge(range: CSVRange, followingRange: CSVRange): void { - const index = this.#random.indexOf(followingRange) - // v8 ignore if -- @preserve - if (index === -1) { - throw new Error('Cannot merge ranges: following range not found in cache') + if ( + this.#averageRowByteCount === 0 + // TODO(SL): instead of a fixed threshold, use a dynamic one based on the number of cached rows? + // or on the variance of the row byte counts? + || Math.abs(averageRowByteCount - this.#averageRowByteCount) / this.#averageRowByteCount > 0.01 + ) { + this.#averageRowByteCount = averageRowByteCount + return true } - range.merge(followingRange) - // remove followingRange from the random ranges - this.#random.splice(index, 1) + + // no significant changes, ignore + return false } /** - * Get the next missing row for the given row range + * Get the cells of a given row * @param options Options - * @param options.rowStart The start row index (0-based, inclusive) - * @param options.rowEnd The end row index (0-based, exclusive) - * @param options.prefixRowsToEstimate Number of average prefix rows to prefetch when estimating (because the first row may be partial). Defaults to 3. - * @returns undefined if no missing row, or an object with: - * - the first byte to start fetching data - * - the first byte to store data (after ignoring some bytes) - * - the expected row number of the first fetched row - */ - getNextMissingRow({ rowStart, rowEnd, prefixRowsToEstimate }: { rowStart: number, rowEnd: number, prefixRowsToEstimate?: number }): { firstByteToFetch: number, firstByteToStore: number, firstRow: number } | undefined { - checkNonNegativeInteger(rowStart) - checkNonNegativeInteger(rowEnd) - prefixRowsToEstimate ??= 3 - checkNonNegativeInteger(prefixRowsToEstimate) - - // try every empty range between cached rows - let cursor = this.#serial.next - - if (cursor.firstByte >= this.#byteLength) { - // No missing row if all rows are cached - return undefined + * @param options.row The row number (0-based) + * @returns The cells of the row, or undefined if the row is not in this range + */ + #getCells({ row }: { row: number }): string[] | undefined { + const cells = this.#cache.serialRange.getRow(row) + if (cells !== undefined) { + return cells } - - for (const nextRange of [...this.#random, { firstRow: Infinity, next: { row: Infinity, firstByte: this.#byteLength } }]) { - if (rowStart < cursor.row) { - // ignore cached rows - rowStart = cursor.row + // find the range containing this row + // try the last range first + for (const [i, range] of this.#cache.randomRanges.reverse().entries()) { + // due to a bug in cosovo?, the last byte of https://huggingface.co/datasets/Mosab-Rezaei/19th-century-novelists/resolve/main/Dataset - Five Authors .csv + // is not counted. To make the demo work, we allow a 1-byte buffer for the last range. + const hotfixBuffer = 1 + const estimatedFirstRow = (i === 0 && range.nextByte >= this.#cache.byteLength - hotfixBuffer && range.rowsCache.numRows > 0) + // special case: last range, and the last stored row is the last row of the file + ? this.numRows - range.rowsCache.numRows + // normal case: estimate based on the byte offset + : this.#guessRowNumberInRandomRange({ byteOffset: range.firstByte }) + if (estimatedFirstRow === undefined) { + return undefined } - if (rowEnd <= rowStart) { - // no missing row (rowEnd is exclusive) - return + const cells = range.getRow(row - estimatedFirstRow) + if (cells !== undefined) { + return cells } - if (rowStart < nextRange.firstRow) { - // the next row to fetch is between the cursor and the next range - if (rowStart === cursor.row || this.averageRowByteCount === undefined) { - // if the requested row is the cursor, we can use its firstByte property directly - // Same if we cannot estimate positions - return { firstByteToFetch: cursor.firstByte, firstByteToStore: cursor.firstByte, firstRow: cursor.row } - } - - // estimate the byte position based on the average row byte count. - const gapRows = rowStart - cursor.row - const gapBytes = Math.round(gapRows * this.averageRowByteCount) - // Start storing here: - const firstByteToStore = Math.max(cursor.firstByte + gapBytes, 0) - // avoid going beyond the end of the file - if (firstByteToStore >= this.#byteLength) { - return undefined - } - - // Start fetching some rows before: - const prefixBytes = Math.round(prefixRowsToEstimate * this.averageRowByteCount) // number of bytes to ignore at the start when estimating - const firstByteToFetch = Math.max(firstByteToStore - prefixBytes, 0) + } + return undefined + } - return { - firstByteToFetch, - firstByteToStore, - firstRow: rowStart, - } - } - // try the next missing range - cursor = nextRange.next + // TODO(SL): look at the ranges to improve the estimation, in particular to avoid gaps between successive rows + #guessRowNumberInRandomRange({ byteOffset }: { byteOffset: number }): number | undefined { + // v8 ignore if -- @preserve + if (this.#averageRowByteCount === undefined) { + // if the cache is complete, there is no random range, so this should not happen + throw new Error('Incoherent state: cannot guess row number in random range when the cache is complete') + } + if (this.#averageRowByteCount === 0) { + // no estimation available + return undefined } + // estimation based on the average row byte count + return Math.max(Math.round((byteOffset - this.#cache.headerByteCount) / this.#averageRowByteCount), 0) } /** - * Check if the given byte range is stored in the cache. - * @param options Options - * @param options.byteOffset The byte offset of the range. - * @returns True if the byte range is stored, false otherwise. + * Get the number of cached rows + * @returns The number of cached rows */ - isStored({ byteOffset }: { byteOffset: number }): boolean { - checkNonNegativeInteger(byteOffset) - - for (const range of [this.#serial, ...this.#random]) { - if (range.firstByte <= byteOffset && byteOffset < range.next.firstByte) { - return true - } - } - - return false + #computeNumCachedRows(): number { + return this.#cache.serialRange.rowsCache.numRows + this.#cache.randomRanges.reduce((sum, range) => sum + range.rowsCache.numRows, 0) } - static fromHeader({ header, byteLength }: { header: ParseResult, byteLength: number }): CSVCache { - return new CSVCache({ - columnNames: header.row, - byteLength, - delimiter: header.meta.delimiter, - newline: header.meta.newline, - headerByteCount: header.meta.byteOffset + header.meta.byteCount, - }) + /** + * Get the number of cached bytes + * @returns The number of cached bytes + */ + #computeNumCachedBytes(): number { + return this.#cache.serialRange.rowsCache.byteCount + this.#cache.randomRanges.reduce((sum, range) => sum + range.rowsCache.byteCount, 0) } } diff --git a/src/dataframe.ts b/src/dataframe.ts index cfb6175..8e86b79 100644 --- a/src/dataframe.ts +++ b/src/dataframe.ts @@ -13,7 +13,7 @@ import { validateGetRowNumberParams, } from 'hightable' -import { CSVCache } from './cache' +import { CSVCache, Estimator } from './cache' import { checkNonNegativeInteger } from './helpers.js' const defaultChunkSize = 100 * 1024 // 100 KB @@ -46,12 +46,8 @@ export async function csvDataFrame(params: Params): Promise { const eventTarget = createEventTarget() const cache = await initializeCSVCachefromURL({ url, byteLength, chunkSize, initialRowCount }) - cache.eventTarget.addEventListener('num-rows-estimate-updated', () => { - // propagate event - eventTarget.dispatchEvent(new CustomEvent('numrowschange')) - }) - // no need to remove the listener, as the cache has the same lifetime as the dataframe - + const estimator = new Estimator({ cache }) + estimator.refresh() const columnDescriptors: DataFrame['columnDescriptors'] = cache.columnNames.map(name => ({ name })) /** @@ -71,14 +67,13 @@ export async function csvDataFrame(params: Params): Promise { column: string orderBy?: OrderBy }): ResolvedValue | undefined { - // until the CSV is fully loaded, we don't know the exact number of rows - const numRows = cache.allRowsCached ? cache.rowCount : Infinity validateGetCellParams({ row, column, orderBy, data: { - numRows, + // until the CSV is fully loaded, we don't know the exact number of rows + numRows: estimator.maxNumRows, columnDescriptors, }, }) @@ -90,7 +85,7 @@ export async function csvDataFrame(params: Params): Promise { // should not happen because of the validation above throw new Error(`Column not found: ${column}`) } - return cache.getCell({ row, column: columnIndex }) + return estimator.getCell({ row, column: columnIndex }) } /** @@ -107,21 +102,22 @@ export async function csvDataFrame(params: Params): Promise { row: number orderBy?: OrderBy }): ResolvedValue | undefined { - // until the CSV is fully loaded, we don't know the exact number of rows - const numRows = cache.allRowsCached ? cache.rowCount : Infinity validateGetRowNumberParams({ row, orderBy, data: { - numRows, + // until the CSV is fully loaded, we don't know the exact number of rows + numRows: estimator.maxNumRows, columnDescriptors, }, }) - return cache.getRowNumber({ row }) + return estimator.getRowNumber({ row }) } /** * Fetch the given range of rows, filling the cache as needed. + * The row numbers are only known exactly for the first range of rows, the rest are estimated. To avoid + * instability, the estimation is updated only at the end of the fetch. * @param options - options * @param options.rowStart - starting row index * @param options.rowEnd - ending row index (exclusive) @@ -144,119 +140,134 @@ export async function csvDataFrame(params: Params): Promise { }): Promise { checkSignal(signal) - // until the CSV is fully loaded, we don't know the exact number of rows - const numRows = cache.allRowsCached ? cache.rowCount : Infinity + const updated = estimator.refresh() + if (updated) { + // propagate event + eventTarget.dispatchEvent(new CustomEvent('numrowschange')) + } + + if (cache.complete) { + // all rows are cached + return + } + validateFetchParams({ rowStart, rowEnd, columns, orderBy, data: { - numRows, + // until the CSV is fully loaded, we don't know the exact number of rows + numRows: Infinity, columnDescriptors, }, }) - if (cache.allRowsCached) { + // Compute the byte range to fetch + for (let r = rowStart; r < rowEnd; r++) { + if (!estimator.isStored({ row: r })) { + break + } + rowStart++ + } + for (let r = rowEnd; r > rowStart; r--) { + if (!estimator.isStored({ row: r - 1 })) { + break + } + rowEnd-- + } + if (rowEnd <= rowStart) { // all rows are already cached return } - const prefixRowsToEstimate = 3 // number of rows to ignore at the start when estimating - const maxLoops = (rowEnd - rowStart + prefixRowsToEstimate) * 10 + 10 // safety to avoid infinite loops + // fetch rows from rowStart to rowEnd (exclusive), with 3 extra rows before and after + const extraRows = 3 + const fetchRowStart = Math.max(0, rowStart - extraRows) + const fetchRowEnd = Math.min(rowEnd + extraRows) - // fetch all missing ranges - let next = cache.getNextMissingRow({ rowStart, rowEnd, prefixRowsToEstimate }) - let j = 0 - while (next) { - // fetch next missing range - j++ - // v8 ignore if -- @preserve - if (j > maxLoops) { - // should not happen - console.debug('Debug info, maximum fetch loops exceeded: ', { rowStart, rowEnd, next, cache }) - throw new Error('Maximum fetch loops exceeded') - } + // we could set initialState to 'default' if firstByte is exactly at the start of a row + // TODO(SL): implement it, by inspecting all the cache ranges, instead of doing a global average + const firstByte = estimator.guessByteOffset({ row: fetchRowStart }) + const lastBytePlusOne = estimator.guessByteOffset({ row: fetchRowEnd }) + if (firstByte === undefined) { + // cannot estimate + return + } + const lastByte = lastBytePlusOne ? lastBytePlusOne - 1 : firstByte - 1 // fetch at least one row - const { firstByteToFetch: maxFirstByteToFetch, firstByteToStore, firstRow } = next - const alignOnChunkSize = maxFirstByteToFetch !== firstByteToStore // if we are estimating, align on chunk size, to save requests - const firstByteToFetch = alignOnChunkSize - ? Math.floor(maxFirstByteToFetch / chunkSize) * chunkSize - : maxFirstByteToFetch - const initialState = firstByteToFetch !== firstByteToStore ? 'detect' : 'default' + const stats = { + parsedRows: 0, + alreadyStored: 0, + newEmpty: 0, + newFull: 0, + ignored: 0, + reachedEOF: false, + } - let k = 0 - let row = firstRow + try { for await (const result of parseURL(url, { delimiter: cache.delimiter, newline: cache.newline, chunkSize, - firstByte: firstByteToFetch, + firstByte, lastByte: byteLength - 1, - initialState, + initialState: 'detect', })) { + stats.parsedRows++ + // Check if the signal has been aborted checkSignal(signal) - k++ - // v8 ignore if -- @preserve - if (k > maxLoops) { - // should not happen - console.debug('Debug info, maximum parse loops exceeded: ', { rowStart, rowEnd, next, cache }) - throw new Error('Maximum parse loops exceeded') - } - if (result.meta.byteOffset < firstByteToStore) { + + if (stats.parsedRows <= 1) { + // we might have started parsing in the middle of a row, ignore this first row + stats.ignored += 1 continue } if (result.meta.byteCount === 0) { // no progress, avoid infinite loop // it's the last line in the file and it's empty - next = undefined + stats.ignored += 1 break } // Store the new row in the cache const isEmpty = isEmptyLine(result.row) - // v8 ignore else -- @preserve - if (!cache.isStored({ byteOffset: result.meta.byteOffset })) { - cache.store({ - cells: isEmpty ? undefined : result.row, - byteOffset: result.meta.byteOffset, - byteCount: result.meta.byteCount, - firstRow: row, - }) - if (!isEmpty && row >= rowStart && row < rowEnd) { - // emit event for newly fetched row within the requested range - eventTarget.dispatchEvent(new CustomEvent('resolve')) - } + + // store if not in the cache yet + const stored = cache.store({ + cells: isEmpty ? undefined : result.row, + byteOffset: result.meta.byteOffset, + byteCount: result.meta.byteCount, + }) + + if (!stored) { + stats.alreadyStored++ } - else { - // the row should not be already stored, but double check - console.debug('Row already stored, should not happen, skipping', { byteOffset: result.meta.byteOffset, row }) + else if (isEmpty) { + stats.newEmpty++ } - if (!isEmpty) { - row++ + else { + eventTarget.dispatchEvent(new CustomEvent('resolve')) + stats.newFull++ } - // next row - next = cache.getNextMissingRow({ rowStart, rowEnd, prefixRowsToEstimate }) - if (!next) { - // no more missing ranges - break + if (result.meta.byteOffset + result.meta.byteCount >= byteLength) { + // end of file + stats.reachedEOF = true } - const nextByte = result.meta.byteOffset + result.meta.byteCount - if (next.firstByteToFetch > nextByte + chunkSize) { - // the next missing range is beyond the current chunk, so we can stop the current loop and start a new fetch + if (result.meta.byteOffset > lastByte) { + // end of the requested range + stats.ignored += 1 break } - // otherwise, continue fetching in the current loop, - // Note that some rows might already be cached. It's ok since fetching takes more time than parsing. } - - // v8 ignore if -- @preserve - if (k === 0) { - // Should not happen - // No progress (no row fetched in this missing range) - // Break to avoid infinite loop - break + } + finally { + // Note: we don't update the estimates after the fetch, to avoid instability during user interactions. + // Exception if the cache is now complete, or we reached the end of the file. + if (cache.complete || stats.reachedEOF === true) { + estimator.refresh() + eventTarget.dispatchEvent(new CustomEvent('numrowschange')) } } } @@ -264,11 +275,11 @@ export async function csvDataFrame(params: Params): Promise { return { metadata: { get isNumRowsEstimated() { - return cache.numRowsEstimate.isEstimate + return estimator.isNumRowsEstimated }, }, get numRows() { - return cache.numRowsEstimate.numRows + return estimator.numRows }, columnDescriptors, getCell, @@ -296,7 +307,7 @@ async function initializeCSVCachefromURL({ url, byteLength, chunkSize, initialRo let cache: CSVCache | undefined = undefined // Fetch the first rows, including the header - let row = 0 + let storedRows = 0 for await (const result of parseURL(url, { chunkSize, lastByte: byteLength - 1 })) { if (cache === undefined) { if (isEmptyLine(result.row, { greedy: true })) { @@ -306,20 +317,22 @@ async function initializeCSVCachefromURL({ url, byteLength, chunkSize, initialRo cache = CSVCache.fromHeader({ header: result, byteLength }) continue } - else if (cache.rowCount >= initialRowCount && result.meta.byteOffset > 0.9 * chunkSize) { + else if (storedRows >= initialRowCount && result.meta.byteOffset > 0.9 * chunkSize) { // enough rows for now break } else { + const isEmpty = isEmptyLine(result.row) // data row - cache.store({ + const stored = cache.store({ // ignore empty lines - cells: isEmptyLine(result.row) ? undefined : result.row, + cells: isEmpty ? undefined : result.row, byteOffset: result.meta.byteOffset, byteCount: result.meta.byteCount, - firstRow: row, }) - row++ + if (stored && !isEmpty) { + storedRows++ + } } } diff --git a/src/helpers.ts b/src/helpers.ts index 859e3c9..b87e012 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -41,3 +41,15 @@ export function checkNonNegativeInteger(value: number): number { } return value } + +/** + * Throws if the provided value is not an integer. + * @param value The desired value. + * @returns The validated value: an integer. + */ +export function checkInteger(value: number): number { + if (!Number.isInteger(value)) { + throw new Error('Value is not an integer') + } + return value +} diff --git a/test/cache.test.ts b/test/cache.test.ts index 2396169..4bffe49 100644 --- a/test/cache.test.ts +++ b/test/cache.test.ts @@ -1,45 +1,82 @@ import { describe, expect, it } from 'vitest' -import { CSVCache, CSVRange } from '../src/cache.js' +import { CSVCache, CSVRange, Estimator, RowsCache } from '../src/cache.js' -describe('CSVRange', () => { +describe('RowsCache', () => { it('should initialize correctly', () => { - const range = new CSVRange({ firstByte: 0, firstRow: 0 }) - expect(range.rowCount).toBe(0) - expect(range.byteCount).toBe(0) - expect(range.firstRow).toBe(0) - expect(range.next).toStrictEqual({ firstByte: 0, row: 0 }) - expect(range.rows).toEqual([]) - expect(range.getCells({ row: 0 })).toBeUndefined() + const rowsCache = new RowsCache() + expect(rowsCache.byteCount).toBe(0) + expect(rowsCache.numRows).toBe(0) + expect(rowsCache.rows).toEqual([]) }) - it('should initialize correctly at a random position', () => { - const range = new CSVRange({ firstByte: 100, firstRow: 10 }) - expect(range.rowCount).toBe(0) - expect(range.byteCount).toBe(0) - expect(range.firstRow).toBe(10) - expect(range.next).toStrictEqual({ firstByte: 100, row: 10 }) - expect(range.rows).toStrictEqual([]) - expect(range.getCells({ row: 10 })).toBeUndefined() + it('should add rows correctly', () => { + const rowsCache = new RowsCache() + rowsCache.append({ + cells: ['d', 'e', 'f'], + byteCount: 10, + }) + rowsCache.prepend({ + cells: ['1', '2', '3'], + byteCount: 10, + }) + expect(rowsCache.byteCount).toBe(20) + expect(rowsCache.numRows).toBe(2) + expect(rowsCache.rows).toEqual([['1', '2', '3'], ['d', 'e', 'f']]) + }) + + it('should merge with another rows cache correctly', () => { + const rowsCache1 = new RowsCache() + rowsCache1.append({ + cells: ['a', 'b', 'c'], + byteCount: 10, + }) + rowsCache1.append({ + cells: ['e', 'f', 'g'], + byteCount: 10, + }) + expect(rowsCache1.numRows).toBe(2) + expect(rowsCache1.byteCount).toBe(20) + + const rowsCache2 = new RowsCache() + rowsCache2.append({ + cells: ['h', 'i', 'j'], + byteCount: 10, + }) + expect(rowsCache2.numRows).toBe(1) + expect(rowsCache2.byteCount).toBe(10) + + rowsCache1.merge(rowsCache2) + + expect(rowsCache1.numRows).toBe(3) + expect(rowsCache1.byteCount).toBe(30) + expect(rowsCache1.rows).toEqual([['a', 'b', 'c'], ['e', 'f', 'g'], ['h', 'i', 'j']]) }) +}) - it('firstRow can be set', () => { - const range = new CSVRange({ firstByte: 0, firstRow: 0 }) - expect(range.firstRow).toBe(0) - range.firstRow = 5 - expect(range.firstRow).toBe(5) +describe('CSVRange', () => { + it('should initialize correctly', () => { + const range = new CSVRange({ firstByte: 0 }) + expect(range.byteCount).toBe(0) + expect(range.nextByte).toBe(0) + expect(range.getRow(0)).toBeUndefined() + expect(range.rowsCache.rows).toEqual([]) + expect(range.rowsCache.byteCount).toBe(0) + expect(range.rowsCache.numRows).toBe(0) }) - it.each([-1, +Infinity, NaN, 1.5])('firstRow setter throws for incorrect value: %d', (value) => { - const range = new CSVRange({ firstByte: 0, firstRow: 0 }) - expect(range.firstRow).toBe(0) - expect(() => { - range.firstRow = value - }).toThrow() + it('should initialize correctly at a random position', () => { + const range = new CSVRange({ firstByte: 100 }) + expect(range.byteCount).toBe(0) + expect(range.nextByte).toBe(100) + expect(range.getRow(0)).toBeUndefined() + expect(range.rowsCache.byteCount).toBe(0) + expect(range.rowsCache.rows).toEqual([]) + expect(range.rowsCache.numRows).toBe(0) }) it('should add rows correctly', () => { - const range = new CSVRange({ firstByte: 100, firstRow: 10 }) + const range = new CSVRange({ firstByte: 100 }) range.append({ byteOffset: 100, byteCount: 10, @@ -58,21 +95,19 @@ describe('CSVRange', () => { byteOffset: 80, byteCount: 10, }) - expect(range.rowCount).toBe(2) expect(range.byteCount).toBe(40) - expect(range.firstRow).toBe(9) - expect(range.next).toStrictEqual({ firstByte: 120, row: 11 }) - expect(range.rows).toEqual([['1', '2', '3'], ['d', 'e', 'f']]) - expect(range.getCells({ row: 7 })).toBeUndefined() - expect(range.getCells({ row: 8 })).toBeUndefined() - expect(range.getCells({ row: 9 })).toEqual(['1', '2', '3']) - expect(range.getCells({ row: 10 })).toEqual(['d', 'e', 'f']) - expect(range.getCells({ row: 11 })).toBeUndefined() - expect(range.getCells({ row: 12 })).toBeUndefined() + expect(range.nextByte).toBe(120) + expect(range.rowsCache.numRows).toBe(2) + expect(range.rowsCache.byteCount).toBe(20) + expect(range.rowsCache.rows).toEqual([['1', '2', '3'], ['d', 'e', 'f']]) + expect(range.getRow(-1)).toBeUndefined() + expect(range.getRow(0)).toEqual(['1', '2', '3']) + expect(range.getRow(1)).toEqual(['d', 'e', 'f']) + expect(range.getRow(2)).toBeUndefined() }) it('should throw when adding non-contiguous rows', () => { - const range = new CSVRange({ firstByte: 100, firstRow: 10 }) + const range = new CSVRange({ firstByte: 100 }) expect(() => { range.prepend({ cells: ['x'], @@ -89,7 +124,7 @@ describe('CSVRange', () => { }) it('should merge with another range correctly', () => { - const range1 = new CSVRange({ firstByte: 0, firstRow: 0 }) + const range1 = new CSVRange({ firstByte: 0 }) range1.append({ byteOffset: 0, byteCount: 10, @@ -99,9 +134,10 @@ describe('CSVRange', () => { byteOffset: 10, byteCount: 10, }) - expect(range1.rowByteCount).toBe(10) + expect(range1.rowsCache.numRows).toBe(1) + expect(range1.rowsCache.byteCount).toBe(10) - const range2 = new CSVRange({ firstByte: 20, firstRow: 2 }) + const range2 = new CSVRange({ firstByte: 20 }) range2.append({ cells: ['e', 'f', 'g'], byteOffset: 20, @@ -111,23 +147,23 @@ describe('CSVRange', () => { byteOffset: 30, byteCount: 10, }) - expect(range2.rowByteCount).toBe(10) + expect(range2.rowsCache.numRows).toBe(1) + expect(range2.rowsCache.byteCount).toBe(10) range1.merge(range2) - expect(range1.rowCount).toBe(2) expect(range1.byteCount).toBe(40) - expect(range1.firstRow).toBe(0) - expect(range1.rowByteCount).toBe(20) - expect(range1.next).toStrictEqual({ firstByte: 40, row: 2 }) - expect(range1.rows).toEqual([['b', 'c', 'd'], ['e', 'f', 'g']]) - expect(range1.getCells({ row: 0 })).toEqual(['b', 'c', 'd']) - expect(range1.getCells({ row: 1 })).toEqual(['e', 'f', 'g']) + expect(range1.nextByte).toBe(40) + expect(range1.rowsCache.numRows).toBe(2) + expect(range1.rowsCache.byteCount).toBe(20) + expect(range1.rowsCache.rows).toEqual([['b', 'c', 'd'], ['e', 'f', 'g']]) + expect(range1.getRow(0)).toEqual(['b', 'c', 'd']) + expect(range1.getRow(1)).toEqual(['e', 'f', 'g']) }) it('should throw when merging non-contiguous ranges', () => { - const range1 = new CSVRange({ firstByte: 0, firstRow: 0 }) - const range2 = new CSVRange({ firstByte: 30, firstRow: 3 }) + const range1 = new CSVRange({ firstByte: 0 }) + const range2 = new CSVRange({ firstByte: 30 }) expect(() => { range1.merge(range2) }).toThrow('Cannot merge ranges: not contiguous') @@ -145,18 +181,11 @@ describe('CSVCache', () => { newline: '\n' as const, }) expect(cache.columnNames).toEqual(['col1', 'col2', 'col3']) - expect(cache.rowCount).toBe(0) + expect(cache.byteLength).toBe(100) + expect(cache.headerByteCount).toBe(15) expect(cache.delimiter).toBe(',') expect(cache.newline).toBe('\n') - expect(cache.averageRowByteCount).toBe(undefined) - expect(cache.headerByteCount).toBe(15) - expect(cache.allRowsCached).toBe(false) - expect(cache.numRowsEstimate).toEqual({ numRows: 0, isEstimate: true }) - expect(cache.getCell({ row: 0, column: 0 })).toBeUndefined() - expect(cache.getRowNumber({ row: 0 })).toBeUndefined() - expect(cache.getNextMissingRow({ rowStart: 0, rowEnd: 10 })).toEqual({ firstByteToStore: 15, firstByteToFetch: 15, firstRow: 0 }) - // As no rows are cached, any range should return the same firstByte - expect(cache.getNextMissingRow({ rowStart: 100, rowEnd: 200 })).toEqual({ firstByteToStore: 15, firstByteToFetch: 15, firstRow: 0 }) + expect(cache.complete).toBe(false) }) it('should initialize from header correctly', () => { @@ -173,15 +202,11 @@ describe('CSVCache', () => { } const cache = CSVCache.fromHeader({ header, byteLength: 100 }) expect(cache.columnNames).toEqual(['col1', 'col2', 'col3']) - expect(cache.rowCount).toBe(0) + expect(cache.byteLength).toBe(100) + expect(cache.headerByteCount).toBe(15) expect(cache.delimiter).toBe(',') expect(cache.newline).toBe('\n') - expect(cache.averageRowByteCount).toBe(undefined) - expect(cache.getCell({ row: 0, column: 0 })).toBeUndefined() - expect(cache.getRowNumber({ row: 0 })).toBeUndefined() - expect(cache.getNextMissingRow({ rowStart: 0, rowEnd: 10 })).toEqual({ firstByteToStore: 15, firstByteToFetch: 15, firstRow: 0 }) - // As no rows are cached, any range should return the same firstByte - expect(cache.getNextMissingRow({ rowStart: 100, rowEnd: 200 })).toEqual({ firstByteToStore: 15, firstByteToFetch: 15, firstRow: 0 }) + expect(cache.complete).toBe(false) }) it.each([ @@ -200,8 +225,8 @@ describe('CSVCache', () => { }) }) - describe('store and retrieve rows', () => { - it('should store and retrieve rows correctly', () => { + describe('when storing rows in order', () => { + it('should append rows to the serial range', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, @@ -209,61 +234,45 @@ describe('CSVCache', () => { delimiter: ',', newline: '\n' as const, }) - // should be row 0 - cache.store({ + const stored1 = cache.store({ cells: ['a', 'b', 'c'], byteOffset: 15, - byteCount: 0, // not forbidden - firstRow: 0, + byteCount: 10, }) - // The average row byte count should be 0 - expect(cache.averageRowByteCount).toBe(0) - // should be row 1 - cache.store({ - cells: ['d', 'e', 'f'], + const stored2 = cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 25, + byteCount: 10, + }) + expect(stored1).toBe(true) + expect(stored2).toBe(true) + expect(cache.complete).toBe(false) + // internal state + expect(cache.serialRange.rowsCache.numRows).toBe(2) + expect(cache.serialRange.byteCount).toBe(35) + expect(cache.randomRanges.length).toBe(0) + }) + it('should account for the bytes of an empty row, but add no cells', () => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 15, + delimiter: ',', + newline: '\n' as const, + }) + const stored = cache.store({ + // no cells byteOffset: 15, - byteCount: 20, - firstRow: 1, - }) - // The average row byte count should be 10 now - expect(cache.averageRowByteCount).toBe(10) - expect(cache.numRowsEstimate).toEqual({ numRows: 9, isEstimate: true }) - // the first row must be retrieved correctly - expect(cache.getCell({ row: 0, column: 0 })).toStrictEqual({ value: 'a' }) - expect(cache.getCell({ row: 0, column: 1 })).toStrictEqual({ value: 'b' }) - expect(cache.getCell({ row: 0, column: 2 })).toStrictEqual({ value: 'c' }) - expect(cache.getRowNumber({ row: 0 })).toStrictEqual({ value: 0 }) - // the second row must be retrieved correctly - expect(cache.getCell({ row: 1, column: 0 })).toStrictEqual({ value: 'd' }) - expect(cache.getCell({ row: 1, column: 1 })).toStrictEqual({ value: 'e' }) - expect(cache.getCell({ row: 1, column: 2 })).toStrictEqual({ value: 'f' }) - expect(cache.getRowNumber({ row: 1 })).toStrictEqual({ value: 1 }) - - // This row should be in a new random range, and the estimated row number should be 3 - cache.store({ - cells: ['d', 'e', 'f'], - byteOffset: 44, - byteCount: 7, - firstRow: 3, - }) - // the average row byte count should be 9 now - expect(cache.averageRowByteCount).toBe(9) - // it should be retrieved correctly with row: 2 - expect(cache.getCell({ row: 3, column: 0 })).toStrictEqual({ value: 'd' }) - expect(cache.getCell({ row: 3, column: 1 })).toStrictEqual({ value: 'e' }) - expect(cache.getCell({ row: 3, column: 2 })).toStrictEqual({ value: 'f' }) - expect(cache.getRowNumber({ row: 3 })).toStrictEqual({ value: 3 }) - - // adding an ignored row should not change the average row byte count - cache.store({ - byteOffset: 60, - byteCount: 5, - firstRow: 4, + byteCount: 10, }) - expect(cache.averageRowByteCount).toBe(9) + expect(stored).toBe(true) + expect(cache.complete).toBe(false) + // internal state + expect(cache.serialRange.rowsCache.numRows).toBe(0) + expect(cache.serialRange.byteCount).toBe(25) + expect(cache.randomRanges.length).toBe(0) }) - - it('should report if all the rows are cached', () => { + it('should report that the cache is complete after storing all the rows', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, @@ -271,235 +280,321 @@ describe('CSVCache', () => { delimiter: ',', newline: '\n' as const, }) - expect(cache.allRowsCached).toBe(false) + expect(cache.complete).toBe(false) // Cache some rows cache.store({ cells: ['a', 'b', 'c'], byteOffset: 10, byteCount: 10, - firstRow: 0, }) cache.store({ cells: ['d', 'e', 'f'], byteOffset: 20, byteCount: 10, - firstRow: 1, }) - expect(cache.allRowsCached).toBe(false) - // Simulate caching all rows by adjusting byteLength and storing a row at the end + expect(cache.complete).toBe(false) + // Last row to complete the cache cache.store({ cells: ['x', 'y', 'z'], byteOffset: 30, byteCount: 70, - firstRow: 2, }) - expect(cache.allRowsCached).toBe(true) - expect(cache.numRowsEstimate).toEqual({ numRows: 3, isEstimate: false }) + expect(cache.complete).toBe(true) }) - - it.each([ - { byteOffset: -10, byteCount: 10 }, - { byteOffset: 10, byteCount: -10 }, - { byteOffset: 100, byteCount: 60 }, - { byteOffset: 95, byteCount: 10 }, - ])('throws if trying to store a row outside of the cache bounds: %o', ({ byteOffset, byteCount }) => { + it('should ignore storing a row inside the serial range', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 15, + delimiter: ',', + newline: '\n' as const, + }) + cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 15, + byteCount: 10, + }) + expect(cache.serialRange.rowsCache.numRows).toBe(1) + expect(cache.serialRange.byteCount).toBe(25) + expect(cache.randomRanges.length).toBe(0) + // store a row inside the serial range + cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 10, + byteCount: 2, + }) + // it did not throw, but also did not change the internal state + expect(cache.serialRange.rowsCache.numRows).toBe(1) + expect(cache.serialRange.byteCount).toBe(25) + expect(cache.randomRanges.length).toBe(0) + }) + it('should throw if trying to store a row that overlaps the serial range', () => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) + cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 15, + byteCount: 10, + }) expect(() => { cache.store({ - cells: ['a', 'b', 'c'], - byteOffset, - byteCount, - firstRow: 0, + cells: ['d', 'e', 'f'], + byteOffset: 20, + byteCount: 10, }) - }).toThrow() + }).toThrowError(/^Cannot store the row/) }) - - it.each([ - { initialRow: { byteOffset: 10, byteCount: 10 }, row: { byteOffset: 5, byteCount: 10 }, expected: 'Cannot store the row: overlap with previous range' }, - { initialRow: { byteOffset: 20, byteCount: 10 }, row: { byteOffset: 5, byteCount: 10 }, expected: 'Cannot store the row: overlap with previous range' }, - { initialRow: { byteOffset: 10, byteCount: 10 }, row: { byteOffset: 15, byteCount: 10 }, expected: 'Cannot store the row: overlap with previous range' }, - { initialRow: { byteOffset: 20, byteCount: 10 }, row: { byteOffset: 15, byteCount: 10 }, expected: 'Cannot store the row: overlap with next range' }, - { initialRow: { byteOffset: 20, byteCount: 10 }, row: { byteOffset: 25, byteCount: 10 }, expected: 'Cannot store the row: overlap with next range' }, - ])('throws when storing rows that overlap existing cached rows: %o', ({ initialRow, row, expected }) => { + }) + describe('when storing a row beyond the first rows', () => { + it('should create a random range', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, - headerByteCount: 10, + byteLength: 100, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store an initial row - cache.store({ ...initialRow, firstRow: 0 }) - expect(() => { - cache.store({ ...row, firstRow: 1 }) - }).toThrow(expected) + const stored = cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 25, + byteCount: 10, + }) + expect(stored).toBe(true) + expect(cache.complete).toBe(false) + // internal state + expect(cache.serialRange.rowsCache.numRows).toBe(0) + expect(cache.serialRange.byteCount).toBe(15) + expect(cache.randomRanges.length).toBe(1) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(1) + expect(cache.randomRanges[0]?.byteCount).toBe(10) }) - - it('should merge two adjacent ranges when storing rows', () => { + it('should append a row to an existing random range if the boundaries match', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store first row cache.store({ cells: ['a', 'b', 'c'], - byteOffset: 10, + byteOffset: 25, byteCount: 10, - firstRow: 0, }) - // Store third row, creating a new random range - cache.store({ - cells: ['e', 'f', 'g'], - byteOffset: 40, - byteCount: 60, - firstRow: 2, - }) - // At this point, we should have two ranges, and not all the rows have been cached - expect(cache.allRowsCached).toBe(false) - expect(cache.rowCount).toBe(2) - // Now store the second row, which should merge the two ranges - cache.store({ + const stored = cache.store({ cells: ['d', 'e', 'f'], - byteOffset: 20, - byteCount: 20, - firstRow: 1, + byteOffset: 35, + byteCount: 10, }) - // now, the cache should include all the rows - expect(cache.allRowsCached).toBe(true) - expect(cache.rowCount).toBe(3) + expect(stored).toBe(true) + expect(cache.complete).toBe(false) + // internal state + expect(cache.serialRange.rowsCache.numRows).toBe(0) + expect(cache.serialRange.byteCount).toBe(15) + expect(cache.randomRanges.length).toBe(1) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(2) + expect(cache.randomRanges[0]?.byteCount).toBe(20) }) - - it('should prepend rows to a random range correctly', () => { + it('should prepend a row to an existing random range if the boundaries match', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store a row to create a random range cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 35, + byteCount: 10, + }) + const stored = cache.store({ cells: ['d', 'e', 'f'], - byteOffset: 40, + byteOffset: 25, byteCount: 10, - firstRow: 3, }) - // Prepend a row to the random range + expect(stored).toBe(true) + expect(cache.complete).toBe(false) + // internal state + expect(cache.serialRange.rowsCache.numRows).toBe(0) + expect(cache.serialRange.byteCount).toBe(15) + expect(cache.randomRanges.length).toBe(1) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(2) + expect(cache.randomRanges[0]?.byteCount).toBe(20) + }) + it('should ignore storing a row inside an existing random range', () => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 15, + delimiter: ',', + newline: '\n' as const, + }) cache.store({ cells: ['a', 'b', 'c'], - byteOffset: 30, + byteOffset: 25, byteCount: 10, - firstRow: 2, }) - // Check that both rows are stored correctly - expect(cache.getCell({ row: 2, column: 0 })).toStrictEqual({ value: 'a' }) - expect(cache.getCell({ row: 2, column: 1 })).toStrictEqual({ value: 'b' }) - expect(cache.getCell({ row: 2, column: 2 })).toStrictEqual({ value: 'c' }) - expect(cache.getCell({ row: 3, column: 0 })).toStrictEqual({ value: 'd' }) - expect(cache.getCell({ row: 3, column: 1 })).toStrictEqual({ value: 'e' }) - expect(cache.getCell({ row: 3, column: 2 })).toStrictEqual({ value: 'f' }) + expect(cache.randomRanges.length).toBe(1) + // Try to store a row inside the random range + cache.store({ + cells: ['d', 'e', 'f'], + byteOffset: 26, + byteCount: 5, + }) + // It did not throw, but also did not change the internal state + expect(cache.randomRanges.length).toBe(1) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(1) + expect(cache.randomRanges[0]?.byteCount).toBe(10) }) - - it('should store rows after checking multiple random ranges', () => { + it('should merge two adjacent ranges when storing rows between', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 300, - headerByteCount: 0, + byteLength: 100, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store first row to create first random range + // Store first row, which creates the first random range cache.store({ - byteOffset: 10, + cells: ['a', 'b', 'c'], + byteOffset: 25, byteCount: 10, - firstRow: 0, }) - // Store second row to create second random range + // Store second row, which creates the second random range cache.store({ - byteOffset: 30, + cells: ['d', 'e', 'f'], + byteOffset: 45, byteCount: 10, - firstRow: 1, }) - // Store third row to create third random range + expect(cache.randomRanges.length).toBe(2) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(1) + expect(cache.randomRanges[0]?.byteCount).toBe(10) + expect(cache.randomRanges[1]?.rowsCache.numRows).toBe(1) + expect(cache.randomRanges[1]?.byteCount).toBe(10) + // Store a row in between, which should merge the two ranges cache.store({ - byteOffset: 50, + cells: ['g', 'h', 'i'], + byteOffset: 35, byteCount: 10, - firstRow: 2, }) + expect(cache.randomRanges.length).toBe(1) + expect(cache.randomRanges[0]?.rowsCache.numRows).toBe(3) + expect(cache.randomRanges[0]?.byteCount).toBe(30) }) - - it('should create a random range between existing ones if there is space', () => { + it('should merge a random range with the serial range when storing preceding rows', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 300, - headerByteCount: 0, + byteLength: 100, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store first row to create first random range - cache.store({ - byteOffset: 10, - byteCount: 10, - firstRow: 0, - }) - // Store third row to create second random range + // Store a row to create a random range cache.store({ - byteOffset: 50, + cells: ['a', 'b', 'c'], + byteOffset: 25, byteCount: 10, - firstRow: 2, }) - // Now store the second row in between + expect(cache.randomRanges.length).toBe(1) + // Now store a row that precedes the random range, which should merge it with the serial range cache.store({ - byteOffset: 30, + cells: ['d', 'e', 'f'], + byteOffset: 15, byteCount: 10, - firstRow: 1, }) + expect(cache.serialRange.rowsCache.numRows).toBe(2) + expect(cache.serialRange.byteCount).toBe(35) + expect(cache.randomRanges.length).toBe(0) }) - }) - - describe('getCell', () => { - it('should throw for out-of-bounds rows or columns', () => { + it('should throw if trying to store a row that partially overlaps with a preceding random range', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - expect(() => cache.getCell({ row: -1, column: 0 })).toThrow() - expect(() => cache.getCell({ row: 0, column: -1 })).toThrow() - expect(() => cache.getCell({ row: 0, column: 3 })).toThrow() + cache.store({ + cells: ['a', 'b', 'c'], + byteOffset: 25, + byteCount: 10, + }) + expect(cache.randomRanges.length).toBe(1) + expect(() => { + cache.store({ + cells: ['d', 'e', 'f'], + byteOffset: 20, + byteCount: 10, + }) + }).toThrowError(/^Cannot store the row/) }) - - it('should return undefined for missing row', () => { + it('should throw if trying to store a row that partially overlaps with a following random range', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 15, delimiter: ',', newline: '\n' as const, }) - // Store a row cache.store({ cells: ['a', 'b', 'c'], - byteOffset: 10, + byteOffset: 25, byteCount: 10, - firstRow: 0, }) - // Missing row - expect(cache.getCell({ row: 1, column: 0 })).toBeUndefined() + expect(cache.randomRanges.length).toBe(1) + expect(() => { + cache.store({ + cells: ['d', 'e', 'f'], + byteOffset: 20, + byteCount: 10, + }) + }).toThrowError(/^Cannot store the row/) }) + }) + it.each([ + { byteOffset: -10, byteCount: 10 }, + { byteOffset: 10, byteCount: -10 }, + { byteOffset: 100, byteCount: 60 }, + { byteOffset: 95, byteCount: 10 }, + ])('throws if trying to store a row outside of the cache bounds or with invalid byteCount: %o', ({ byteOffset, byteCount }) => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 10, + delimiter: ',', + newline: '\n' as const, + }) + expect(() => { + cache.store({ + cells: ['a', 'b', 'c'], + byteOffset, + byteCount, + }) + }).toThrow() + }) +}) - it('should return empty string for missing column', () => { +describe('Estimator', () => { + describe('constructor', () => { + it('should initialize correctly', () => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 15, + delimiter: ',', + newline: '\n' as const, + }) + const estimator = new Estimator({ cache }) + expect(estimator.numRows).toBe(0) + expect(estimator.isNumRowsEstimated).toBe(true) + }) + it('should estimate on the contents of the cache only after calling refresh()', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, @@ -507,18 +602,19 @@ describe('CSVCache', () => { delimiter: ',', newline: '\n' as const, }) - // Store a row cache.store({ - cells: ['a', 'b'], // missing last column byteOffset: 10, byteCount: 10, - firstRow: 0, + cells: ['a', 'b', 'c'], }) - // Missing column - expect(cache.getCell({ row: 0, column: 2 })).toStrictEqual({ value: '' }) + const estimator = new Estimator({ cache }) + expect(estimator.numRows).toBe(0) + expect(estimator.isNumRowsEstimated).toBe(true) + estimator.refresh() + expect(estimator.numRows).toBe(9) + expect(estimator.isNumRowsEstimated).toBe(true) }) - - it('should return the correct cell value for existing rows and columns', () => { + it('can be copied from another estimator, and then be independent', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, @@ -526,52 +622,59 @@ describe('CSVCache', () => { delimiter: ',', newline: '\n' as const, }) - // Store a row cache.store({ - cells: ['a', 'b', 'c'], byteOffset: 10, byteCount: 10, - firstRow: 0, + cells: ['a', 'b', 'c'], }) - // Existing row and columns - expect(cache.getCell({ row: 0, column: 0 })).toStrictEqual({ value: 'a' }) - expect(cache.getCell({ row: 0, column: 1 })).toStrictEqual({ value: 'b' }) - expect(cache.getCell({ row: 0, column: 2 })).toStrictEqual({ value: 'c' }) + const estimator = new Estimator({ cache }) + estimator.refresh() + + const copy = estimator.copy() + expect(copy.numRows).toBe(9) + expect(copy.isNumRowsEstimated).toBe(true) + + // only two rows, the second one is empty + cache.store({ + byteOffset: 20, + byteCount: 80, + }) + estimator.refresh() + expect(estimator.numRows).toBe(1) + expect(estimator.isNumRowsEstimated).toBe(false) + expect(copy.numRows).toBe(9) + expect(copy.isNumRowsEstimated).toBe(true) }) }) - describe('getRowNumber', () => { - it('should throw for out-of-bounds rows', () => { + describe('refresh', () => { + it('updates the internal state if the cache is now complete', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 100, delimiter: ',', newline: '\n' as const, }) - expect(() => cache.getRowNumber({ row: -1 })).toThrow() + const estimator = new Estimator({ cache }) + const updated = estimator.refresh() + expect(updated).toBe(true) }) - - it('should return undefined for missing row', () => { + it('does not update the internal state if the cache was already complete', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 100, delimiter: ',', newline: '\n' as const, }) - // Store a row - cache.store({ - cells: ['a', 'b', 'c'], - byteOffset: 10, - byteCount: 10, - firstRow: 0, - }) - // Missing row - expect(cache.getRowNumber({ row: 1 })).toBeUndefined() + const estimator = new Estimator({ cache }) + estimator.refresh() + // refresh again + const updated = estimator.refresh() + expect(updated).toBe(false) }) - - it('should return the correct row number for existing rows', () => { + it('updates the internal state if the previous average row byte count was zero', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, @@ -579,204 +682,233 @@ describe('CSVCache', () => { delimiter: ',', newline: '\n' as const, }) - // Store a row cache.store({ - cells: ['a', 'b', 'c'], byteOffset: 10, byteCount: 10, - firstRow: 0, + cells: ['a', 'b', 'c'], }) - // Existing row - expect(cache.getRowNumber({ row: 0 })).toStrictEqual({ value: 0 }) + const estimator = new Estimator({ cache }) + const updated = estimator.refresh() + expect(updated).toBe(true) }) - }) - - describe('getNextMissingRow', () => { - it.each([ - { range: { rowStart: 0, rowEnd: 10 }, expectedFirstRow: 0 }, - { range: { rowStart: 5, rowEnd: 15 }, expectedFirstRow: 0 }, - { range: { rowStart: 10, rowEnd: 20 }, expectedFirstRow: 0 }, - ])('should propose the first byte if the cache is empty since it cannot estimate positions: %o', ({ range, expectedFirstRow }) => { + it('updates the internal state if the difference with the previous average row byte count is significant (more than 1%)', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, + byteLength: 100, headerByteCount: 10, delimiter: ',', newline: '\n' as const, }) - expect(cache.getNextMissingRow(range)).toEqual({ firstByteToFetch: 10, firstByteToStore: 10, firstRow: expectedFirstRow }) + cache.store({ + byteOffset: 10, + byteCount: 10, + cells: ['a', 'b', 'c'], + }) + const estimator = new Estimator({ cache }) + estimator.refresh() + cache.store({ + byteOffset: 20, + byteCount: 70, + cells: ['a', 'b', 'c'], + }) + const updated = estimator.refresh() + expect(updated).toBe(true) }) - - it.each([ - { options: { rowStart: 0, rowEnd: 0 }, expected: undefined }, - { options: { rowStart: 0, rowEnd: 2 }, expected: { firstByteToFetch: 20, firstByteToStore: 20, firstRow: 1 } }, - { options: { rowStart: 1, rowEnd: 2 }, expected: { firstByteToFetch: 20, firstByteToStore: 20, firstRow: 1 } }, - { options: { rowStart: 2, rowEnd: 2 }, expected: undefined }, - { options: { rowStart: 3, rowEnd: 2 }, expected: undefined }, - ])('should propose the next missing row correctly after #serial range: %o', ({ options, expected }) => { + it('does not update the internal state if the difference with the previous average row byte count is not significant (less than 1%)', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, - headerByteCount: 10, + byteLength: 1000, + headerByteCount: 0, delimiter: ',', newline: '\n' as const, }) - // Stored row is in the #serial range, next.firstByte is 20 - cache.store({ byteOffset: 10, byteCount: 10, cells: ['x', 'y', 'z'], firstRow: 0 }) - expect(cache.getNextMissingRow(options)).toEqual(expected) + cache.store({ + byteOffset: 0, + byteCount: 400, + cells: ['a', 'b', 'c'], + }) + const estimator = new Estimator({ cache }) + estimator.refresh() + cache.store({ + byteOffset: 400, + byteCount: 401, + cells: ['a', 'b', 'c'], + }) + const updated = estimator.refresh() + expect(updated).toBe(false) }) + }) - it.each([ - { options: { rowStart: 0, rowEnd: 4 }, expected: { firstByteToStore: 10, firstByteToFetch: 10, firstRow: 0 } }, - { options: { rowStart: 1, rowEnd: 4 }, expected: { firstByteToStore: 20, firstByteToFetch: 0, firstRow: 1 } }, - { options: { rowStart: 2, rowEnd: 4 }, expected: { firstByteToStore: 40, firstByteToFetch: 40, firstRow: 3 } }, - { options: { rowStart: 3, rowEnd: 4 }, expected: { firstByteToStore: 40, firstByteToFetch: 40, firstRow: 3 } }, - { options: { rowStart: 4, rowEnd: 5 }, expected: { firstByteToStore: 50, firstByteToFetch: 20, firstRow: 4 } }, - ])('should propose the next missing row correctly around a random range: %o', ({ options, expected }) => { + describe('max row number, used for validation', () => { + it('should be Infinity if the number of rows is an estimate', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, + byteLength: 100, headerByteCount: 10, delimiter: ',', newline: '\n' as const, }) - // Stored row is in a random range, next.firstByte is 40 - cache.store({ byteOffset: 30, byteCount: 10, cells: ['x', 'y', 'z'], firstRow: 2 }) - // the average row byte count is now 10 - expect(cache.averageRowByteCount).toBe(10) - // the estimated row number is 2 - expect(cache.getRowNumber({ row: 2 })).toEqual({ value: 2 }) - - expect(cache.getNextMissingRow(options)).toEqual(expected) + const estimator = new Estimator({ cache }) + estimator.refresh() + expect(estimator.maxNumRows).toBe(Infinity) + expect(estimator.isNumRowsEstimated).toBe(true) }) - - it('should return undefined when all rows are cached', () => { + it('should be numRows if the number of rows is not an estimate', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 100, delimiter: ',', newline: '\n' as const, }) - // Simulate caching all rows by storing a row that covers the entire byteLength - cache.store({ - cells: ['x', 'y', 'z'], - byteOffset: 10, - byteCount: 90, - firstRow: 0, - }) - expect(cache.allRowsCached).toBe(true) - expect(cache.getNextMissingRow({ rowStart: 0, rowEnd: 10 })).toBeUndefined() + const estimator = new Estimator({ cache }) + estimator.refresh() + expect(estimator.maxNumRows).toBe(0) + expect(estimator.maxNumRows).toBe(estimator.numRows) + expect(estimator.isNumRowsEstimated).toBe(false) }) + }) - it('should return undefined when all rows until rowEnd - 1 is already cached (exclusive)', () => { + describe('isStored, getRowNumber, getCell and guessByteOffset', () => { + it('return nothing for any row when the cache is empty', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, + byteLength: 100, headerByteCount: 10, delimiter: ',', newline: '\n' as const, }) - // Store rows 0, 1, and 2 - cache.store({ byteOffset: 10, byteCount: 10, cells: ['a', 'b', 'c'], firstRow: 0 }) // row 0 - cache.store({ byteOffset: 20, byteCount: 10, cells: ['d', 'e', 'f'], firstRow: 1 }) // row 1 - cache.store({ byteOffset: 30, byteCount: 10, cells: ['g', 'h', 'i'], firstRow: 2 }) // row 2 + const estimator = new Estimator({ cache }) + expect(estimator.isStored({ row: 0 })).toBe(false) + expect(estimator.isStored({ row: 10 })).toBe(false) + expect(estimator.isStored({ row: 100 })).toBe(false) - expect(cache.getNextMissingRow({ rowStart: 2, rowEnd: 2 })).toBeUndefined() - expect(cache.getNextMissingRow({ rowStart: 2, rowEnd: 3 })).toBeUndefined() - expect(cache.getNextMissingRow({ rowStart: 2, rowEnd: 4 })).toStrictEqual({ firstByteToStore: 40, firstByteToFetch: 40, firstRow: 3 }) - }) - }) + expect(estimator.getRowNumber({ row: 0 })).toBeUndefined() + expect(estimator.getRowNumber({ row: 10 })).toBeUndefined() + expect(estimator.getRowNumber({ row: 100 })).toBeUndefined() + + expect(estimator.getCell({ row: 0, column: 0 })).toBeUndefined() + expect(estimator.getCell({ row: 10, column: 1 })).toBeUndefined() + expect(estimator.getCell({ row: 100, column: 2 })).toBeUndefined() + expect(() => estimator.getCell({ row: 0, column: 3 })).toThrowError(/^Column index/) - describe('isStored', () => { - it('should correctly identify stored and non-stored rows', () => { + // The first byte offset is after the header + expect(estimator.guessByteOffset({ row: 0 })).toEqual(10) + // No estimation available + expect(estimator.guessByteOffset({ row: 1 })).toBeUndefined() + expect(estimator.guessByteOffset({ row: 10 })).toBeUndefined() + expect(estimator.guessByteOffset({ row: 100 })).toBeUndefined() + }) + it('returns the correct value for rows stored at the start (exact match)', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], byteLength: 100, - headerByteCount: 10, + headerByteCount: 0, delimiter: ',', newline: '\n' as const, }) - // Store a row cache.store({ cells: ['a', 'b', 'c'], - byteOffset: 10, + byteOffset: 0, byteCount: 10, - firstRow: 0, - }) - cache.store({ - byteOffset: 60, - byteCount: 10, - firstRow: 1, }) cache.store({ cells: ['a', 'b', 'c'], - byteOffset: 80, + byteOffset: 10, byteCount: 10, - firstRow: 1, }) - expect(cache.isStored({ byteOffset: 10 })).toBe(true) - expect(cache.isStored({ byteOffset: 20 })).toBe(false) - expect(cache.isStored({ byteOffset: 60 })).toBe(true) - expect(cache.isStored({ byteOffset: 80 })).toBe(true) - }) - }) - - describe('updateRowEstimates', () => { - it('should update row estimates correctly', () => { + const estimator = new Estimator({ cache }) + estimator.refresh() + expect(estimator.isStored({ row: 0 })).toBe(true) + expect(estimator.isStored({ row: 1 })).toBe(true) + expect(estimator.isStored({ row: 2 })).toBe(false) + + expect(estimator.getRowNumber({ row: 0 })).toEqual({ value: 0 }) + expect(estimator.getRowNumber({ row: 1 })).toEqual({ value: 1 }) + // getRowNumber returns a value if it can estimate it, even if the row is not stored + expect(estimator.getRowNumber({ row: 2 })).toEqual({ value: 2 }) + // getRowNumber returns undefined for rows way beyond the estimated number of rows + expect(estimator.getRowNumber({ row: -1 })).toBeUndefined() + expect(estimator.getRowNumber({ row: 1000 })).toBeUndefined() + + expect(estimator.getCell({ row: 0, column: 0 })).toEqual({ value: 'a' }) + expect(estimator.getCell({ row: 1, column: 0 })).toEqual({ value: 'a' }) + expect(estimator.getCell({ row: 2, column: 0 })).toBeUndefined() + + // just after the first rows (exact) + expect(estimator.guessByteOffset({ row: 2 })).toEqual(20) + // beyond the first rows (estimated) + expect(estimator.guessByteOffset({ row: 3 })).toEqual(30) + }) + it('returns the correct value for rows stored in the middle of the file (estimated match)', () => { const cache = new CSVCache({ columnNames: ['col1', 'col2', 'col3'], - byteLength: 200, - headerByteCount: 10, + byteLength: 100, + headerByteCount: 0, delimiter: ',', newline: '\n' as const, }) - // Initially, averageRowByteCount should be undefined - expect(cache.averageRowByteCount).toBeUndefined() - expect(cache.rowCount).toBe(0) - // Update changes nothing as there are no rows - cache.updateRowEstimates() - expect(cache.averageRowByteCount).toBeUndefined() - expect(cache.rowCount).toBe(0) - // Store an empty row cache.store({ + cells: ['a', 'b', 'c'], byteOffset: 10, byteCount: 10, - firstRow: 0, - }) - expect(cache.averageRowByteCount).toBeUndefined() - expect(cache.rowCount).toBe(0) - // Update changes nothing as there are no stored rows with cells - cache.updateRowEstimates() - expect(cache.averageRowByteCount).toBeUndefined() - expect(cache.rowCount).toBe(0) - // Store one row + }) cache.store({ - cells: ['a', 'b', 'c'], + // missing last cell + cells: ['a', 'b'], byteOffset: 20, byteCount: 10, - firstRow: 0, }) - expect(cache.averageRowByteCount).toBe(10) - expect(cache.rowCount).toBe(1) - // Store another row at a random position + const estimator = new Estimator({ cache }) + estimator.refresh() + expect(estimator.isStored({ row: 0 })).toBe(false) + expect(estimator.isStored({ row: 1 })).toBe(true) + expect(estimator.isStored({ row: 2 })).toBe(true) + expect(estimator.isStored({ row: 3 })).toBe(false) + + // getRowNumber returns a value if it can estimate it, even if the row is not stored + expect(estimator.getRowNumber({ row: 0 })).toEqual({ value: 0 }) + expect(estimator.getRowNumber({ row: 1 })).toEqual({ value: 1 }) + expect(estimator.getRowNumber({ row: 2 })).toEqual({ value: 2 }) + expect(estimator.getRowNumber({ row: 3 })).toEqual({ value: 3 }) + // getRowNumber returns undefined for rows way beyond the estimated number of rows + expect(estimator.getRowNumber({ row: -1 })).toBeUndefined() + expect(estimator.getRowNumber({ row: 1000 })).toBeUndefined() + + expect(estimator.getCell({ row: 0, column: 0 })).toBeUndefined() + expect(estimator.getCell({ row: 1, column: 0 })).toEqual({ value: 'a' }) + expect(estimator.getCell({ row: 2, column: 0 })).toEqual({ value: 'a' }) + // the missing cell should be returned as empty + expect(estimator.getCell({ row: 2, column: 2 })).toEqual({ value: '' }) + expect(estimator.getCell({ row: 3, column: 0 })).toBeUndefined() + + // at the start (exact) + expect(estimator.guessByteOffset({ row: 0 })).toEqual(0) + // just after the first estimated rows (estimated) + expect(estimator.guessByteOffset({ row: 3 })).toEqual(30) + // beyond the estimated rows (estimated) + expect(estimator.guessByteOffset({ row: 8 })).toEqual(80) + }) + it('returns nothing if the estimator was not refreshed yet', () => { + const cache = new CSVCache({ + columnNames: ['col1', 'col2', 'col3'], + byteLength: 100, + headerByteCount: 0, + delimiter: ',', + newline: '\n' as const, + }) cache.store({ - cells: ['d', 'e', 'f'], - byteOffset: 50, - byteCount: 70, - firstRow: 3, - }) - expect(cache.rowCount).toBe(2) - // its row index is estimated to be 3, because it has been computed with the current average (10) - expect(cache.getRowNumber({ row: 3 })).toEqual({ value: 3 }) - // but the averageRowByteCount should be updated correctly - expect(cache.averageRowByteCount).toBe(40) - // Now, update the row estimates - cache.updateRowEstimates() - // The averageRowByteCount should remain the same - expect(cache.averageRowByteCount).toBe(40) - // but the row index should be updated to 2 - expect(cache.getRowNumber({ row: 2 })).toEqual({ value: 2 }) + cells: ['a', 'b', 'c'], + byteOffset: 10, + byteCount: 10, + }) + const estimator = new Estimator({ cache }) + // not refreshed yet + expect(estimator.isStored({ row: 0 })).toBe(false) + expect(estimator.getRowNumber({ row: 0 })).toBeUndefined() + expect(estimator.getCell({ row: 0, column: 0 })).toBeUndefined() + expect(estimator.guessByteOffset({ row: 0 })).toBe(0) + expect(estimator.guessByteOffset({ row: 1 })).toBeUndefined() }) }) }) diff --git a/test/dataframe.test.ts b/test/dataframe.test.ts index e41e4c4..539d2f9 100644 --- a/test/dataframe.test.ts +++ b/test/dataframe.test.ts @@ -254,7 +254,7 @@ describe('csvDataFrame', () => { revoke() }) - it('should return undefined for not yet cached rows', async () => { + it('should return an estimated value when possible for not yet cached rows', async () => { const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n' const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ @@ -263,7 +263,7 @@ describe('csvDataFrame', () => { initialRowCount: 2, chunkSize: 5, }) - expect(df.getRowNumber({ row: 2 })).toBeUndefined() + expect(df.getRowNumber({ row: 2 })).toEqual({ value: 2 }) revoke() }) @@ -371,14 +371,14 @@ describe('csvDataFrame', () => { revoke() }) - it('should throw when fetching out-of-bound rows, if the dataframe is fully loaded', async () => { + it('does not throw when fetching out-of-bound rows, if the dataframe is fully loaded', async () => { const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n' const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, byteLength: fileSize, }) - await expect(df.fetch?.({ rowStart: 5, rowEnd: 10 })).rejects.toThrow() + await expect(df.fetch?.({ rowStart: 5, rowEnd: 10 })).resolves.toBeUndefined() revoke() }) @@ -398,24 +398,33 @@ describe('csvDataFrame', () => { }) it('should fetch rows at a random position if requested', async () => { - const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n' + // padding the strings to have more consistent row sizes + const text = 'a\n' + Array(100).fill(0).map((_, i) => i.toString().padStart(2, '0')).join('\n') + '\n' const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, byteLength: fileSize, - initialRowCount: 2, + initialRowCount: 5, chunkSize: 5, }) - expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 3, rowEnd: 4 }) - // row 3 is now cached, while row 2 is still not cached - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '10' }) - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() + expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '00' }) + expect(df.getCell({ row: 30, column: 'a' })).toBeUndefined() + await df.fetch?.({ rowStart: 30, rowEnd: 31 }) + // row 30 is now cached + expect(df.getRowNumber({ row: 30 })).toStrictEqual({ value: 30 }) + expect(df.getCell({ row: 30, column: 'a' })).toStrictEqual({ value: '30' }) + // some rows around too + expect(df.getCell({ row: 29, column: 'a' })).toStrictEqual({ value: '29' }) + expect(df.getCell({ row: 31, column: 'a' })).toStrictEqual({ value: '31' }) + // but not far away + expect(df.getCell({ row: 20, column: 'a' })).toBeUndefined() + expect(df.getCell({ row: 40, column: 'a' })).toBeUndefined() + revoke() }) it('will fetch an incorrect row if the average row size had been overestimated', async () => { - const text = 'a,b,c\n111,222,333\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n' + const text = 'a\n' + '0'.repeat(10) + '\n' + Array(99).fill(0).map((_, i) => (i + 1).toString().padStart(2, '0')).join('\n') const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, @@ -423,126 +432,51 @@ describe('csvDataFrame', () => { initialRowCount: 1, chunkSize: 5, }) - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 2, rowEnd: 5 }) - expect(df.getCell({ row: 2, column: 'a' })).toStrictEqual({ value: '10' }) // should be 7 - expect(df.getCell({ row: 3, column: 'b' })).toStrictEqual({ value: '14' }) // should be 11 - expect(df.getCell({ row: 4, column: 'c' })).toBeUndefined() // should be 15 + expect(df.getCell({ row: 10, column: 'a' })).toBeUndefined() + await df.fetch?.({ rowStart: 10, rowEnd: 11 }) + expect(df.getCell({ row: 10, column: 'a' })).toStrictEqual({ value: '27' }) // should be 10 + // fetch again, which refreshes the average row size + await df.fetch?.({ rowStart: 10, rowEnd: 11 }) + expect(df.getCell({ row: 10, column: 'a' })).toStrictEqual({ value: '08' }) // should be 10 + // fetch again, which refreshes the average row size, but does not fetch more rows + await df.fetch?.({ rowStart: 10, rowEnd: 11 }) + expect(df.getCell({ row: 10, column: 'a' })).toStrictEqual({ value: '08' }) // should be 10 revoke() }) - it('fails to fetch the last rows if the average row size has been overestimated', async () => { - const text = 'a,b,c\n111111111,222222222,333333333\n,4,5,6\n7,8,9\n10,11,12\n13,14,15\n' + it('dispatches one "resolve" event per stored row in the range', async () => { + const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n' const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, byteLength: fileSize, - initialRowCount: 1, + initialRowCount: 2, chunkSize: 5, }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 3, rowEnd: 5 }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'b' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'c' })).toBeUndefined() - revoke() - }) - - it('should break the current parsing and start a new one if the next row is beyond one chunk size', async () => { - const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n16,17,18\n' - const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) - const df = await csvDataFrame({ - url, - byteLength: fileSize, - initialRowCount: 1, - chunkSize: 8, // small chunk size to force multiple fetches - }) - expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '1' }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 4, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 5, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 2, rowEnd: 5 }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toStrictEqual({ value: '7' }) - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '10' }) - expect(df.getCell({ row: 4, column: 'a' })).toStrictEqual({ value: '13' }) - expect(df.getCell({ row: 5, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 1, rowEnd: 6 }) - expect(df.getCell({ row: 1, column: 'a' })).toStrictEqual({ value: '4' }) - expect(df.getCell({ row: 5, column: 'a' })).toStrictEqual({ value: '16' }) - revoke() - }) - it('fetches incorrect rows if the row estimation is incorrect', async () => { - const text = 'a,b,c\n111111,222222,333333\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n16,17,18\n19,20,21\n22,23,24\n' - const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) - const df = await csvDataFrame({ - url, - byteLength: fileSize, - initialRowCount: 1, - chunkSize: 5, + let resolveEventCount = 0 + df.eventTarget?.addEventListener('resolve', () => { + resolveEventCount++ }) - expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '111111' }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 4, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 5, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 6, column: 'a' })).toBeUndefined() - // average row size here is 21, because of the first row - - await df.fetch?.({ rowStart: 2, rowEnd: 6 }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - // erroneously got row 4 instead of row 2, due to the overestimation of the average row size - expect(df.getCell({ row: 2, column: 'a' })).toStrictEqual({ value: '13' }) - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '16' }) - expect(df.getCell({ row: 4, column: 'a' })).toStrictEqual({ value: '19' }) - expect(df.getCell({ row: 5, column: 'a' })).toStrictEqual({ value: '22' }) - expect(df.getCell({ row: 6, column: 'a' })).toBeUndefined() - revoke() - }) - - it('does not fetch all the rows, if the row estimation is incorrect', async () => { - const text = 'a,b,c\n111111,222222,333333\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n16,17,18\n19,20,21\n22,23,24\n' - const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) - const df = await csvDataFrame({ - url, - byteLength: fileSize, - initialRowCount: 1, - chunkSize: 5, - }) - expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '111111' }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 4, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 5, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 6, column: 'a' })).toBeUndefined() - // average row size here is 21, because of the first row + await df.fetch?.({ rowStart: 2, rowEnd: 5 }) + expect(resolveEventCount).toBe(3) - await df.fetch?.({ rowStart: 2, rowEnd: 7 }) + // No event because rows are already resolved + await df.fetch?.({ rowStart: 2, rowEnd: 5 }) + expect(resolveEventCount).toBe(3) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - // erroneously got row 4 instead of row 2, due to the overestimation of the average row size - expect(df.getCell({ row: 2, column: 'a' })).toStrictEqual({ value: '13' }) - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '16' }) - expect(df.getCell({ row: 4, column: 'a' })).toStrictEqual({ value: '19' }) - expect(df.getCell({ row: 5, column: 'a' })).toStrictEqual({ value: '22' }) - // the last row, row 6, was not fetched, even if it was requested - expect(df.getCell({ row: 6, column: 'a' })).toBeUndefined() revoke() }) - it('dispatches one "resolve" event per stored row in the range', async () => { - const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n' + it('does nothing if trying to fetch random rows, and no initial rows are cached', async () => { + const text = 'aaaaaaaaaa\n' + Array(100).fill(0).map((_, i) => i.toString().padStart(2, '0')).join('\n') const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, byteLength: fileSize, - initialRowCount: 2, + // no initial rows! -> no estimate + initialRowCount: 0, chunkSize: 5, }) @@ -551,46 +485,45 @@ describe('csvDataFrame', () => { resolveEventCount++ }) - await df.fetch?.({ rowStart: 2, rowEnd: 4 }) - expect(resolveEventCount).toBe(2) + // no estimate -> does nothing + await df.fetch?.({ rowStart: 30, rowEnd: 31 }) + expect(resolveEventCount).toBe(0) - // No event because rows are already resolved - await df.fetch?.({ rowStart: 2, rowEnd: 4 }) - expect(resolveEventCount).toBe(2) + // the first row can always be fetched + await df.fetch?.({ rowStart: 0, rowEnd: 5 }) + // note that only one row has actually been fetched + // TODO(SL): be smarter when we fetch from the start + expect(resolveEventCount).toBe(1) - await df.fetch?.({ rowStart: 2, rowEnd: 6 }) - expect(resolveEventCount).toBe(3) + // now, the offset for row 30 can be estimated, and rows can be fetched + await df.fetch?.({ rowStart: 30, rowEnd: 31 }) + expect(resolveEventCount).toBe(8) revoke() }) - it('can parse again the same rows, if the chunk has been fetched but some rows were already cached', async () => { - const text = 'a,b,c\n1,2,3\n4,5,6\n7,8,9\n10,11,12\n13,14,15\n' + it('does fetches only what is needed', async () => { + const text = 'aa\n' + Array(100).fill(0).map((_, i) => i.toString().padStart(2, '0')).join('\n') const { url, revoke, fileSize } = toBlobURL(text, { withNodeWorkaround: true }) const df = await csvDataFrame({ url, byteLength: fileSize, - initialRowCount: 1, + initialRowCount: 20, chunkSize: 5, }) - expect(df.getCell({ row: 0, column: 'a' })).toStrictEqual({ value: '1' }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 4, column: 'a' })).toBeUndefined() - await df.fetch?.({ rowStart: 3, rowEnd: 4 }) - expect(df.getCell({ row: 1, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 2, column: 'a' })).toBeUndefined() - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '10' }) - expect(df.getCell({ row: 4, column: 'a' })).toBeUndefined() + let resolveEventCount = 0 + df.eventTarget?.addEventListener('resolve', () => { + resolveEventCount++ + }) - // Fetch again the same chunk - await df.fetch?.({ rowStart: 1, rowEnd: 5 }) - expect(df.getCell({ row: 1, column: 'a' })).toStrictEqual({ value: '4' }) - expect(df.getCell({ row: 2, column: 'a' })).toStrictEqual({ value: '7' }) - expect(df.getCell({ row: 3, column: 'a' })).toStrictEqual({ value: '10' }) - expect(df.getCell({ row: 4, column: 'a' })).toStrictEqual({ value: '13' }) + // fetch the last rows + await df.fetch?.({ rowStart: 80, rowEnd: 100 }) + expect(resolveEventCount).toBe(22) + + // fetch all the rows: only the missing rows should be fetched + await df.fetch?.({ rowStart: 0, rowEnd: 100 }) + expect(resolveEventCount).toBe(80) revoke() })