From 0a43d2b570d1eb6449bdbf426234d8721e9be5cb Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sat, 4 Apr 2026 04:15:40 -0600 Subject: [PATCH 1/4] fix: show only preferred engine in README benchmark table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark report showed both native and WASM build speeds side-by-side in the README, but since total build time includes many engine-independent JS stages (insert, resolve, structure, roles), the numbers appeared misleadingly similar (~13.3 vs ~13.6 ms/file). This was inconsistent with how rebuild and query metrics already used only the preferred engine. Now build speed, query time, and the 50k estimate all use `pref` (native when available, WASM fallback) — matching the pattern for all other rows. Detailed per-engine breakdown remains in BUILD-BENCHMARKS.md. --- scripts/update-benchmark-report.ts | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/scripts/update-benchmark-report.ts b/scripts/update-benchmark-report.ts index 69f1f0ce..ce0252d5 100644 --- a/scripts/update-benchmark-report.ts +++ b/scripts/update-benchmark-report.ts @@ -324,21 +324,17 @@ if (prev) { if (fs.existsSync(readmePath)) { let readme = fs.readFileSync(readmePath, 'utf8'); - // Build the table rows — show both engines when native is available - // Pick the preferred engine: native when available, WASM as fallback + // Pick the preferred engine: native when available, WASM as fallback. + // Show only one engine — total build time includes many engine-independent + // JS stages (insert, resolve, structure, roles) that dilute the native + // parsing advantage, making side-by-side numbers misleadingly similar. + // Detailed per-engine breakdown lives in BUILD-BENCHMARKS.md. const pref = latest.native || latest.wasm; const prefLabel = latest.native ? ' (native)' : ''; let rows = ''; - if (latest.native) { - rows += `| Build speed (native) | **${latest.native.perFile.buildTimeMs} ms/file** |\n`; - rows += `| Build speed (WASM) | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; - rows += `| Query time (native) | **${formatMs(latest.native.queryTimeMs)}** |\n`; - rows += `| Query time (WASM) | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; - } else { - rows += `| Build speed | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; - rows += `| Query time | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; - } + rows += `| Build speed${prefLabel} | **${pref.perFile.buildTimeMs} ms/file** |\n`; + rows += `| Query time${prefLabel} | **${formatMs(pref.queryTimeMs)}** |\n`; // Incremental rebuild rows (prefer native, fallback to WASM) if (pref.noopRebuildMs != null) { @@ -354,10 +350,8 @@ if (fs.existsSync(readmePath)) { if (pref.queries.pathMs != null) rows += `| Query: path | **${pref.queries.pathMs}ms** |\n`; } - // 50k-file estimate - const estBuild = latest.native - ? formatMs(latest.native.perFile.buildTimeMs * ESTIMATE_FILES) - : formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES); + // 50k-file estimate (uses preferred engine) + const estBuild = formatMs(pref.perFile.buildTimeMs * ESTIMATE_FILES); rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estBuild} build** |\n`; // Preserve existing benchmark link line from README rather than hardcoding. From ca6a82830fb3745b4887b8101e64939c621e6cff Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sat, 4 Apr 2026 14:45:45 -0600 Subject: [PATCH 2/4] fix: guard wasm null access and add prefLabel to query rows Add optional chaining on latest.wasm.perFile (line 203) to prevent TypeError when only native benchmark data is present. Also append prefLabel to Query: fn-deps and Query: path rows for consistent engine labeling in the README table. --- scripts/update-benchmark-report.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/update-benchmark-report.ts b/scripts/update-benchmark-report.ts index ce0252d5..12c2b3a4 100644 --- a/scripts/update-benchmark-report.ts +++ b/scripts/update-benchmark-report.ts @@ -200,11 +200,11 @@ md += '| Metric | Native (Rust) | WASM |\n'; md += '|--------|---:|---:|\n'; const estNative = latest.native?.perFile; -const estWasm = latest.wasm.perFile; -md += `| Build time | ${estNative ? formatMs(estNative.buildTimeMs * ESTIMATE_FILES) : 'n/a'} | ${formatMs(estWasm.buildTimeMs * ESTIMATE_FILES)} |\n`; -md += `| DB size | ${estNative ? formatBytes(estNative.dbSizeBytes * ESTIMATE_FILES) : 'n/a'} | ${formatBytes(estWasm.dbSizeBytes * ESTIMATE_FILES)} |\n`; -md += `| Nodes | ${estNative ? Math.round(estNative.nodes * ESTIMATE_FILES).toLocaleString() : 'n/a'} | ${Math.round(estWasm.nodes * ESTIMATE_FILES).toLocaleString()} |\n`; -md += `| Edges | ${estNative ? Math.round(estNative.edges * ESTIMATE_FILES).toLocaleString() : 'n/a'} | ${Math.round(estWasm.edges * ESTIMATE_FILES).toLocaleString()} |\n\n`; +const estWasm = latest.wasm?.perFile; +md += `| Build time | ${estNative ? formatMs(estNative.buildTimeMs * ESTIMATE_FILES) : 'n/a'} | ${estWasm ? formatMs(estWasm.buildTimeMs * ESTIMATE_FILES) : 'n/a'} |\n`; +md += `| DB size | ${estNative ? formatBytes(estNative.dbSizeBytes * ESTIMATE_FILES) : 'n/a'} | ${estWasm ? formatBytes(estWasm.dbSizeBytes * ESTIMATE_FILES) : 'n/a'} |\n`; +md += `| Nodes | ${estNative ? Math.round(estNative.nodes * ESTIMATE_FILES).toLocaleString() : 'n/a'} | ${estWasm ? Math.round(estWasm.nodes * ESTIMATE_FILES).toLocaleString() : 'n/a'} |\n`; +md += `| Edges | ${estNative ? Math.round(estNative.edges * ESTIMATE_FILES).toLocaleString() : 'n/a'} | ${estWasm ? Math.round(estWasm.edges * ESTIMATE_FILES).toLocaleString() : 'n/a'} |\n\n`; // ── Incremental Rebuilds section ────────────────────────────────────────── const hasIncremental = history.some( @@ -346,8 +346,8 @@ if (fs.existsSync(readmePath)) { // Query latency rows (pick two representative queries, skip if null) if (pref.queries) { - if (pref.queries.fnDepsMs != null) rows += `| Query: fn-deps | **${pref.queries.fnDepsMs}ms** |\n`; - if (pref.queries.pathMs != null) rows += `| Query: path | **${pref.queries.pathMs}ms** |\n`; + if (pref.queries.fnDepsMs != null) rows += `| Query: fn-deps${prefLabel} | **${pref.queries.fnDepsMs}ms** |\n`; + if (pref.queries.pathMs != null) rows += `| Query: path${prefLabel} | **${pref.queries.pathMs}ms** |\n`; } // 50k-file estimate (uses preferred engine) From 97b2cf48176aed6ea243abee2cb3e08b91d774b1 Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sat, 4 Apr 2026 16:05:24 -0600 Subject: [PATCH 3/4] fix: show both engines in all README benchmark rows --- scripts/update-benchmark-report.ts | 90 +++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 27 deletions(-) diff --git a/scripts/update-benchmark-report.ts b/scripts/update-benchmark-report.ts index 12c2b3a4..2099f672 100644 --- a/scripts/update-benchmark-report.ts +++ b/scripts/update-benchmark-report.ts @@ -324,35 +324,63 @@ if (prev) { if (fs.existsSync(readmePath)) { let readme = fs.readFileSync(readmePath, 'utf8'); - // Pick the preferred engine: native when available, WASM as fallback. - // Show only one engine — total build time includes many engine-independent - // JS stages (insert, resolve, structure, roles) that dilute the native - // parsing advantage, making side-by-side numbers misleadingly similar. - // Detailed per-engine breakdown lives in BUILD-BENCHMARKS.md. - const pref = latest.native || latest.wasm; - const prefLabel = latest.native ? ' (native)' : ''; + // Show both engines side-by-side when native is available + const hasNative = latest.native != null; let rows = ''; - rows += `| Build speed${prefLabel} | **${pref.perFile.buildTimeMs} ms/file** |\n`; - rows += `| Query time${prefLabel} | **${formatMs(pref.queryTimeMs)}** |\n`; - - // Incremental rebuild rows (prefer native, fallback to WASM) - if (pref.noopRebuildMs != null) { - rows += `| No-op rebuild${prefLabel} | **${formatMs(pref.noopRebuildMs)}** |\n`; + if (hasNative) { + rows += `| Build speed | **${latest.native.perFile.buildTimeMs} ms/file** | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; + rows += `| Query time | **${formatMs(latest.native.queryTimeMs)}** | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; + } else { + rows += `| Build speed | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; + rows += `| Query time | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; } - if (pref.oneFileRebuildMs != null) { - rows += `| 1-file rebuild${prefLabel} | **${formatMs(pref.oneFileRebuildMs)}** |\n`; + + // Incremental rebuild rows + if (hasNative) { + const nativeNoop = latest.native.noopRebuildMs != null ? `**${formatMs(latest.native.noopRebuildMs)}**` : 'n/a'; + const wasmNoop = latest.wasm.noopRebuildMs != null ? `**${formatMs(latest.wasm.noopRebuildMs)}**` : 'n/a'; + if (latest.native.noopRebuildMs != null || latest.wasm.noopRebuildMs != null) { + rows += `| No-op rebuild | ${nativeNoop} | ${wasmNoop} |\n`; + } + const nativeOneFile = latest.native.oneFileRebuildMs != null ? `**${formatMs(latest.native.oneFileRebuildMs)}**` : 'n/a'; + const wasmOneFile = latest.wasm.oneFileRebuildMs != null ? `**${formatMs(latest.wasm.oneFileRebuildMs)}**` : 'n/a'; + if (latest.native.oneFileRebuildMs != null || latest.wasm.oneFileRebuildMs != null) { + rows += `| 1-file rebuild | ${nativeOneFile} | ${wasmOneFile} |\n`; + } + } else { + if (latest.wasm.noopRebuildMs != null) { + rows += `| No-op rebuild | **${formatMs(latest.wasm.noopRebuildMs)}** |\n`; + } + if (latest.wasm.oneFileRebuildMs != null) { + rows += `| 1-file rebuild | **${formatMs(latest.wasm.oneFileRebuildMs)}** |\n`; + } } - // Query latency rows (pick two representative queries, skip if null) - if (pref.queries) { - if (pref.queries.fnDepsMs != null) rows += `| Query: fn-deps${prefLabel} | **${pref.queries.fnDepsMs}ms** |\n`; - if (pref.queries.pathMs != null) rows += `| Query: path${prefLabel} | **${pref.queries.pathMs}ms** |\n`; + // Query latency rows (pick two representative queries) + if (hasNative) { + const nq = latest.native.queries; + const wq = latest.wasm.queries; + if (nq?.fnDepsMs != null || wq?.fnDepsMs != null) { + rows += `| Query: fn-deps | **${nq?.fnDepsMs ?? 'n/a'}ms** | **${wq?.fnDepsMs ?? 'n/a'}ms** |\n`; + } + if (nq?.pathMs != null || wq?.pathMs != null) { + rows += `| Query: path | **${nq?.pathMs ?? 'n/a'}ms** | **${wq?.pathMs ?? 'n/a'}ms** |\n`; + } + } else if (latest.wasm.queries) { + if (latest.wasm.queries.fnDepsMs != null) rows += `| Query: fn-deps | **${latest.wasm.queries.fnDepsMs}ms** |\n`; + if (latest.wasm.queries.pathMs != null) rows += `| Query: path | **${latest.wasm.queries.pathMs}ms** |\n`; } - // 50k-file estimate (uses preferred engine) - const estBuild = formatMs(pref.perFile.buildTimeMs * ESTIMATE_FILES); - rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estBuild} build** |\n`; + // 50k-file estimate + if (hasNative) { + const estNativeBuild = formatMs(latest.native.perFile.buildTimeMs * ESTIMATE_FILES); + const estWasmBuild = formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES); + rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estNativeBuild} build** | **~${estWasmBuild} build** |\n`; + } else { + const estBuild = formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES); + rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estBuild} build** |\n`; + } // Preserve existing benchmark link line from README rather than hardcoding. // Fall back to a default if we can't find it. @@ -363,8 +391,8 @@ if (fs.existsSync(readmePath)) { } // Resolution precision/recall — from resolution-benchmark.ts JSON merged into entry + // Resolution is engine-independent, so show single value (span both columns when needed) if (latest.resolution) { - // Compute aggregate precision/recall across all languages const langs = Object.values(latest.resolution); if (langs.length > 0) { const totalResolved = langs.reduce((s, l) => s + l.totalResolved, 0); @@ -372,19 +400,27 @@ if (fs.existsSync(readmePath)) { const totalTP = langs.reduce((s, l) => s + l.truePositives, 0); const aggPrecision = totalResolved > 0 ? `${((totalTP / totalResolved) * 100).toFixed(1)}%` : 'n/a'; const aggRecall = totalExpected > 0 ? `${((totalTP / totalExpected) * 100).toFixed(1)}%` : 'n/a'; - rows += `| Resolution precision | **${aggPrecision}** |\n`; - rows += `| Resolution recall | **${aggRecall}** |\n`; + if (hasNative) { + rows += `| Resolution precision | **${aggPrecision}** | — |\n`; + rows += `| Resolution recall | **${aggRecall}** | — |\n`; + } else { + rows += `| Resolution precision | **${aggPrecision}** |\n`; + rows += `| Resolution recall | **${aggRecall}** |\n`; + } } } + const tableHeader = hasNative + ? `| Metric | Native | WASM |\n|---|---|---|` + : `| Metric | Latest |\n|---|---|`; + const perfSection = `## 📊 Performance Self-measured on every release via CI (${benchmarkLinks}): *Last updated: v${latest.version} (${latest.date})* -| Metric | Latest | -|---|---| +${tableHeader} ${rows} Metrics are normalized per file for cross-version comparability. Times above are for a full initial build — incremental rebuilds only re-parse changed files. `; From 8a83fe52afe61d951c17d104785414fd5bab67ae Mon Sep 17 00:00:00 2001 From: carlos-alm <127798846+carlos-alm@users.noreply.github.com> Date: Sun, 5 Apr 2026 00:20:27 -0600 Subject: [PATCH 4/4] fix: guard against null wasm data in README benchmark table Use a `hasBoth` flag (native && wasm both present) instead of just `hasNative` for the two-column layout. When only native data exists without WASM, falls back to single-column layout instead of crashing with a TypeError on null wasm access. --- scripts/update-benchmark-report.ts | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/scripts/update-benchmark-report.ts b/scripts/update-benchmark-report.ts index 2099f672..f7a343ac 100644 --- a/scripts/update-benchmark-report.ts +++ b/scripts/update-benchmark-report.ts @@ -324,20 +324,25 @@ if (prev) { if (fs.existsSync(readmePath)) { let readme = fs.readFileSync(readmePath, 'utf8'); - // Show both engines side-by-side when native is available + // Show both engines side-by-side when both are available; + // fall back to native-only or WASM-only single-column layout otherwise. const hasNative = latest.native != null; + const hasBoth = hasNative && latest.wasm != null; let rows = ''; - if (hasNative) { + if (hasBoth) { rows += `| Build speed | **${latest.native.perFile.buildTimeMs} ms/file** | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; rows += `| Query time | **${formatMs(latest.native.queryTimeMs)}** | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; + } else if (hasNative) { + rows += `| Build speed | **${latest.native.perFile.buildTimeMs} ms/file** |\n`; + rows += `| Query time | **${formatMs(latest.native.queryTimeMs)}** |\n`; } else { rows += `| Build speed | **${latest.wasm.perFile.buildTimeMs} ms/file** |\n`; rows += `| Query time | **${formatMs(latest.wasm.queryTimeMs)}** |\n`; } // Incremental rebuild rows - if (hasNative) { + if (hasBoth) { const nativeNoop = latest.native.noopRebuildMs != null ? `**${formatMs(latest.native.noopRebuildMs)}**` : 'n/a'; const wasmNoop = latest.wasm.noopRebuildMs != null ? `**${formatMs(latest.wasm.noopRebuildMs)}**` : 'n/a'; if (latest.native.noopRebuildMs != null || latest.wasm.noopRebuildMs != null) { @@ -349,16 +354,17 @@ if (fs.existsSync(readmePath)) { rows += `| 1-file rebuild | ${nativeOneFile} | ${wasmOneFile} |\n`; } } else { - if (latest.wasm.noopRebuildMs != null) { - rows += `| No-op rebuild | **${formatMs(latest.wasm.noopRebuildMs)}** |\n`; + const pref = latest.native || latest.wasm; + if (pref.noopRebuildMs != null) { + rows += `| No-op rebuild | **${formatMs(pref.noopRebuildMs)}** |\n`; } - if (latest.wasm.oneFileRebuildMs != null) { - rows += `| 1-file rebuild | **${formatMs(latest.wasm.oneFileRebuildMs)}** |\n`; + if (pref.oneFileRebuildMs != null) { + rows += `| 1-file rebuild | **${formatMs(pref.oneFileRebuildMs)}** |\n`; } } // Query latency rows (pick two representative queries) - if (hasNative) { + if (hasBoth) { const nq = latest.native.queries; const wq = latest.wasm.queries; if (nq?.fnDepsMs != null || wq?.fnDepsMs != null) { @@ -367,18 +373,20 @@ if (fs.existsSync(readmePath)) { if (nq?.pathMs != null || wq?.pathMs != null) { rows += `| Query: path | **${nq?.pathMs ?? 'n/a'}ms** | **${wq?.pathMs ?? 'n/a'}ms** |\n`; } - } else if (latest.wasm.queries) { - if (latest.wasm.queries.fnDepsMs != null) rows += `| Query: fn-deps | **${latest.wasm.queries.fnDepsMs}ms** |\n`; - if (latest.wasm.queries.pathMs != null) rows += `| Query: path | **${latest.wasm.queries.pathMs}ms** |\n`; + } else { + const pref = latest.native || latest.wasm; + if (pref.queries?.fnDepsMs != null) rows += `| Query: fn-deps | **${pref.queries.fnDepsMs}ms** |\n`; + if (pref.queries?.pathMs != null) rows += `| Query: path | **${pref.queries.pathMs}ms** |\n`; } // 50k-file estimate - if (hasNative) { + if (hasBoth) { const estNativeBuild = formatMs(latest.native.perFile.buildTimeMs * ESTIMATE_FILES); const estWasmBuild = formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES); rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estNativeBuild} build** | **~${estWasmBuild} build** |\n`; } else { - const estBuild = formatMs(latest.wasm.perFile.buildTimeMs * ESTIMATE_FILES); + const pref = latest.native || latest.wasm; + const estBuild = formatMs(pref.perFile.buildTimeMs * ESTIMATE_FILES); rows += `| ~${(ESTIMATE_FILES).toLocaleString()} files (est.) | **~${estBuild} build** |\n`; } @@ -400,7 +408,7 @@ if (fs.existsSync(readmePath)) { const totalTP = langs.reduce((s, l) => s + l.truePositives, 0); const aggPrecision = totalResolved > 0 ? `${((totalTP / totalResolved) * 100).toFixed(1)}%` : 'n/a'; const aggRecall = totalExpected > 0 ? `${((totalTP / totalExpected) * 100).toFixed(1)}%` : 'n/a'; - if (hasNative) { + if (hasBoth) { rows += `| Resolution precision | **${aggPrecision}** | — |\n`; rows += `| Resolution recall | **${aggRecall}** | — |\n`; } else { @@ -410,7 +418,7 @@ if (fs.existsSync(readmePath)) { } } - const tableHeader = hasNative + const tableHeader = hasBoth ? `| Metric | Native | WASM |\n|---|---|---|` : `| Metric | Latest |\n|---|---|`;