Precompile MDX wiki files to .mdx.tsx with build integration, testing, and performance validation

.gitattributes

@@ -2,3 +2,4 @@
 /.yarn/releases/*    binary
 /.yarn/plugins/**/*  binary
 /.pnp.*              binary linguist-generated
+*.mdx.tsx            linguist-generated

.github/actions/setup-tools/action.yml

@@ -9,8 +9,21 @@ runs:
       with:
         auto-install: true
 
+    - uses: actions/cache@v4
+      with:
+        path: .yarn/cache
+        key: yarn-cache-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}
+        restore-keys: |
+          yarn-cache-${{ runner.os }}-
+
     - name: Yarn install
       run: yarn install
+      env:
+        # Using a local cache and Github caching saves ~10s
+        YARN_CACHE_FOLDER: .yarn/cache
+        YARN_ENABLE_GLOBAL_CACHE: "0"
+        # Run hardened mode only if the PR head repo is a fork, saves ~10s
+        YARN_ENABLE_HARDENED_MODE: ${{ github.event.pull_request.head.repo.fork && '1' || '0' }}
       shell: bash
 
     - name: Install ffmpeg (for tests)

.prettierignore

@@ -8,6 +8,7 @@ pnpm-lock.yaml
 /projects/app/.vercel
 /projects/app/src/assets/**/*.svg
 /projects/app/src/assets/**/*.json
+/projects/app/src/**/*.mdx.tsx
 /projects/*/dist
 *.asset.json
 *.m4a

.yarnrc.yml

@@ -1,6 +1,3 @@
-injectEnvironmentFiles:
-  - .env.yarn
-
 nodeLinker: node-modules
 nmMode: hardlinks-local # Speed boost
 

moon.yml

@@ -2,7 +2,6 @@ id: root
 
 fileGroups:
   yarnConfig:
-    - ".env.yarn"
     - ".yarnrc.yml"
     - "package.json"
     - "projects/*/package.json"

projects/app/bin/benchmarkMdxCompilation.ts

@@ -0,0 +1,170 @@
+import { glob, readFile } from "@pinyinly/lib/fs";
+import { transform } from "@pinyinly/mdx/transformer";
+import makeDebug from "debug";
+import path from "node:path";
+
+const debug = makeDebug(`pyly:benchmark`);
+
+/**
+ * Benchmark MDX compilation approaches to validate if precompilation is faster
+ * than on-the-fly compilation during builds and tests.
+ */
+async function main() {
+  debug(`🏁 Starting MDX compilation benchmark...`);
+
+  // Find all MDX files in the wiki directory
+  const mdxFiles = await glob(
+    path.join(process.cwd(), `projects/app/src/client/wiki/**/*.mdx`),
+  );
+
+  // Take a reasonable sample for benchmarking to avoid long test times
+  const sampleSize = Math.min(100, mdxFiles.length);
+  const sampleFiles = mdxFiles.slice(0, sampleSize);
+
+  debug(
+    `📄 Benchmarking with ${sampleFiles.length} MDX files (sample of ${mdxFiles.length} total)`,
+  );
+
+  // Benchmark 1: On-the-fly compilation (simulating current approach)
+  debug(`\n🔄 Benchmark 1: On-the-fly compilation`);
+  const onTheFlyStart = Date.now();
+
+  let onTheFlySuccess = 0;
+  let onTheFlyErrors = 0;
+
+  for (const mdxFile of sampleFiles) {
+    try {
+      const mdxContent = await readFile(mdxFile, `utf-8`);
+      await transform({
+        filename: mdxFile,
+        src: mdxContent,
+      });
+      onTheFlySuccess++;
+    } catch {
+      onTheFlyErrors++;
+    }
+  }
+
+  const onTheFlyDuration = Date.now() - onTheFlyStart;
+
+  // Benchmark 2: Reading precompiled files (current precompilation approach)
+  debug(`\n🔄 Benchmark 2: Reading precompiled .mdx.tsx files`);
+  const precompiledStart = Date.now();
+
+  let precompiledSuccess = 0;
+  let precompiledErrors = 0;
+
+  for (const mdxFile of sampleFiles) {
+    try {
+      const compiledFile = mdxFile.replace(/\.mdx$/, `.mdx.tsx`);
+      await readFile(compiledFile, `utf-8`);
+      precompiledSuccess++;
+    } catch {
+      precompiledErrors++;
+    }
+  }
+
+  const precompiledDuration = Date.now() - precompiledStart;
+
+  // Benchmark 3: Precompilation time (one-time cost)
+  debug(`\n🔄 Benchmark 3: Precompilation time for sample`);
+  const precompilationStart = Date.now();
+
+  let precompilationSuccess = 0;
+  let precompilationErrors = 0;
+
+  for (const mdxFile of sampleFiles) {
+    try {
+      const mdxContent = await readFile(mdxFile, `utf-8`);
+      await transform({
+        filename: mdxFile,
+        src: mdxContent,
+      });
+      precompilationSuccess++;
+    } catch {
+      precompilationErrors++;
+    }
+  }
+
+  const precompilationDuration = Date.now() - precompilationStart;
+
+  // Results
+  debug(`\n📊 Benchmark Results:`);
+  debug(`\n1. On-the-fly compilation:`);
+  debug(`   ⏱️  Duration: ${onTheFlyDuration}ms`);
+  debug(`   ✅ Success: ${onTheFlySuccess} files`);
+  debug(`   ❌ Errors: ${onTheFlyErrors} files`);
+  debug(
+    `   📈 Avg per file: ${(onTheFlyDuration / sampleFiles.length).toFixed(2)}ms`,
+  );
+
+  debug(`\n2. Reading precompiled files:`);
+  debug(`   ⏱️  Duration: ${precompiledDuration}ms`);
+  debug(`   ✅ Success: ${precompiledSuccess} files`);
+  debug(`   ❌ Errors: ${precompiledErrors} files`);
+  debug(
+    `   📈 Avg per file: ${(precompiledDuration / sampleFiles.length).toFixed(2)}ms`,
+  );
+
+  debug(`\n3. Precompilation time (one-time cost):`);
+  debug(`   ⏱️  Duration: ${precompilationDuration}ms`);
+  debug(`   ✅ Success: ${precompilationSuccess} files`);
+  debug(`   ❌ Errors: ${precompilationErrors} files`);
+  debug(
+    `   📈 Avg per file: ${(precompilationDuration / sampleFiles.length).toFixed(2)}ms`,
+  );
+
+  // Analysis
+  const speedupFactor = onTheFlyDuration / precompiledDuration;
+  const breakEvenPoint = Math.ceil(
+    precompilationDuration / (onTheFlyDuration - precompiledDuration),
+  );
+
+  debug(`\n🔍 Analysis:`);
+  debug(`   🚀 Speedup factor: ${speedupFactor.toFixed(2)}x faster`);
+  debug(`   ⚖️  Break-even point: ${breakEvenPoint} uses of compiled files`);
+
+  if (speedupFactor > 1) {
+    debug(
+      `   ✅ Precompilation approach is ${speedupFactor.toFixed(2)}x faster for runtime usage`,
+    );
+  } else {
+    debug(`   ⚠️  On-the-fly compilation might be comparable or faster`);
+  }
+
+  // Extrapolate to full dataset
+  const fullDatasetOnTheFly =
+    (onTheFlyDuration / sampleFiles.length) * mdxFiles.length;
+  const fullDatasetPrecompiled =
+    (precompiledDuration / sampleFiles.length) * mdxFiles.length;
+  const fullDatasetPrecompilation =
+    (precompilationDuration / sampleFiles.length) * mdxFiles.length;
+
+  debug(`\n📈 Extrapolated to full dataset (${mdxFiles.length} files):`);
+  debug(
+    `   On-the-fly: ~${Math.round(fullDatasetOnTheFly)}ms (${(fullDatasetOnTheFly / 1000).toFixed(1)}s)`,
+  );
+  debug(
+    `   Precompiled reading: ~${Math.round(fullDatasetPrecompiled)}ms (${(fullDatasetPrecompiled / 1000).toFixed(1)}s)`,
+  );
+  debug(
+    `   Precompilation cost: ~${Math.round(fullDatasetPrecompilation)}ms (${(fullDatasetPrecompilation / 1000).toFixed(1)}s)`,
+  );
+
+  const totalSavings = fullDatasetOnTheFly - fullDatasetPrecompiled;
+  debug(
+    `   💰 Savings per usage: ~${Math.round(totalSavings)}ms (${(totalSavings / 1000).toFixed(1)}s)`,
+  );
+
+  debug(`\n🎯 Recommendation:`);
+  if (speedupFactor > 2) {
+    debug(`   ✅ Precompilation provides significant performance benefit`);
+  } else if (speedupFactor > 1.2) {
+    debug(`   ✅ Precompilation provides moderate performance benefit`);
+  } else {
+    debug(`   ⚠️  Performance benefit of precompilation may be minimal`);
+  }
+}
+
+// Run the benchmark
+await main();

projects/app/bin/codegenMdx.ts

@@ -0,0 +1,93 @@
+import { glob, mkdir, readFile, writeFile } from "@pinyinly/lib/fs";
+import { transform } from "@pinyinly/mdx/transformer";
+import makeDebug from "debug";
+import path from "node:path";
+
+const debug = makeDebug(`pyly`);
+
+/**
+ * Enhanced template that adds proper TypeScript types for precompiled components
+ */
+function getTypedTemplate(rawMdxString: string): string {
+  return (
+    // Add @ts-nocheck for generated files to avoid JSX runtime issues
+    `// @ts-nocheck\n` +
+    rawMdxString
+      // Replace the function signature to include proper types
+      .replace(
+        /function _createMdxContent\(props\)/,
+        `function _createMdxContent(props: any)`,
+      )
+      // Replace the default export to include proper types
+      .replace(
+        /export default function MDXContent\(props = \{\}\)/,
+        `export default function MDXContent(props: any = {})`,
+      )
+  );
+}
+
+/**
+ * Precompile all wiki MDX files to TSX files
+ * This avoids runtime compilation during builds and tests
+ */
+async function main() {
+  debug(`🔄 Starting wiki MDX precompilation...`);
+
+  const startTime = Date.now();
+
+  // Find all MDX files in the src directory
+  const mdxFiles = await glob(path.join(process.cwd(), `src/**/*.mdx`));
+
+  debug(`📄 Found ${mdxFiles.length} MDX files to precompile`);
+
+  let compiled = 0;
+  let errors = 0;
+
+  for (const mdxFile of mdxFiles) {
+    try {
+      // Read the MDX file
+      const mdxContent = await readFile(mdxFile, `utf-8`);
+
+      // Transform it to TSX
+      const { src: rawTsxContent } = await transform({
+        filename: mdxFile,
+        src: mdxContent,
+      });
+
+      // Apply additional typing for precompiled components
+      const tsxContent = getTypedTemplate(rawTsxContent);
+
+      // Generate the corresponding .mdx.tsx file path
+      const tsxFile = mdxFile.replace(/\.mdx$/, `.mdx.tsx`);
+
+      // Ensure the directory exists
+      await mkdir(path.dirname(tsxFile), { recursive: true });
+
+      // Write the TSX file
+      await writeFile(tsxFile, tsxContent, `utf-8`);
+
+      compiled++;
+
+      if (compiled % 100 === 0) {
+        debug(`Compiled ${compiled}/${mdxFiles.length} files...`);
+      }
+    } catch (error) {
+      console.error(`❌ Error compiling ${mdxFile}:`, error);
+      errors++;
+    }
+  }
+
+  const duration = Date.now() - startTime;
+
+  debug(`\n🎉 Precompilation complete!`);
+  debug(`   ✅ Compiled: ${compiled} files`);
+  debug(`   ❌ Errors: ${errors} files`);
+  debug(`   ⏱️ Duration: ${duration}ms`);
+
+  if (errors > 0) {
+    throw new Error(`Finished with errors.`);
+  }
+}
+
+// Run the precompilation
+await main();

projects/app/bin/tsconfig.json

@@ -6,9 +6,10 @@
     "moduleResolution": "NodeNext",
     "lib": ["ES2024"],
     "jsx": "react-jsx",
+    "rootDir": "../../..",
     "outDir": "../../../.moon/cache/types/projects/app-bin"
   },
-  "include": ["**/*", "../src/types/**/*.ts"],
+  "include": ["**/*"],
   "references": [
     {
       "path": "../src"

projects/app/eslint.config.mjs

@@ -6,6 +6,7 @@ import {
   includeIgnoreFile,
   plugins,
 } from "@pinyinly/eslint-rules";
+import { mdxRecommended } from "@pinyinly/mdx/eslint";
 import queryPlugin from "@tanstack/eslint-plugin-query";
 import drizzlePlugin from "eslint-plugin-drizzle";
 import { builtinModules } from "node:module";
@@ -29,6 +30,7 @@ export const pluginsConfig = {
 // Based on https://github.com/typescript-eslint/typescript-eslint/blob/41323746de299e6d62b4d6122975301677d7c8e0/eslint.config.mjs
 export default defineConfig(
   gitignoreConfig,
+  mdxRecommended,
 
   pluginsConfig,