Fix/c manifest and extraction (#139)

* Fixed definesToKeep can have undefined

* Gave signature/definition linking responsibility to includeResolver

* Removed leftover logging

* Added caching to include resolution

* Changed internal inclusion structure

* Changed structure for included symbols

* Switched invocations to IncludedSymbol

* Added first-level include recursion management

* Removed includes for deleted files

* Includedirs and compaction

* Extraction refinement

* Fixed bad handling of typedefs with same name

* Improved recursive inclusion resolution

* Added credits

Author: LeBoufty

packages/cli/src/config/localConfig.ts

@@ -25,6 +25,11 @@ export const localConfigSchema = z.object({
         .optional(),
     })
     .optional(), // python specific config
+  [cLanguage]: z
+    .object({
+      includedirs: z.array(z.string()).optional(),
+    })
+    .optional(), // c specific config
   project: z.object({
     include: z.array(z.string()),
     exclude: z.array(z.string()).optional(),

packages/cli/src/languagePlugins/c/dependencyFormatting/index.ts

@@ -25,10 +25,14 @@ export class CDependencyFormatter {
 
   constructor(
     files: Map<string, { path: string; rootNode: Parser.SyntaxNode }>,
+    includeDirs: string[] = [],
   ) {
     this.symbolRegistry = new CSymbolRegistry(files);
     this.#registry = this.symbolRegistry.getRegistry();
-    this.includeResolver = new CIncludeResolver(this.symbolRegistry);
+    this.includeResolver = new CIncludeResolver(
+      this.symbolRegistry,
+      includeDirs,
+    );
     this.invocationResolver = new CInvocationResolver(this.includeResolver);
   }
 
@@ -60,7 +64,7 @@ export class CDependencyFormatter {
     const dependencies: Record<string, CDependency> = {};
     const resolved = fileDependencies.resolved;
     for (const [symName, symbol] of resolved) {
-      const filepath = symbol.declaration.filepath;
+      const filepath = symbol.symbol.declaration.filepath;
       const id = symName;
       if (!dependencies[filepath]) {
         dependencies[filepath] = {

packages/cli/src/languagePlugins/c/extractor/index.test.ts

@@ -1,16 +1,23 @@
 import { describe, test } from "@std/testing/bdd";
 import { expect } from "@std/expect";
-import { cFilesFolder, getCFilesContentMap } from "../testFiles/index.ts";
+import {
+  cFilesFolder,
+  dummyLocalConfig,
+  getCFilesContentMap,
+} from "../testFiles/index.ts";
 import { CExtractor } from "./index.ts";
 import { join } from "@std/path";
 import { generateCDependencyManifest } from "../../../manifest/dependencyManifest/c/index.ts";
 
 describe("CExtractor", () => {
   const cContentMap = getCFilesContentMap();
-  const manifest = generateCDependencyManifest(cContentMap);
-  const extractor = new CExtractor(cContentMap, manifest);
+  const manifest = generateCDependencyManifest(cContentMap, dummyLocalConfig);
+  const extractor = new CExtractor(cContentMap, manifest, dummyLocalConfig);
   const burgers = join(cFilesFolder, "burgers.h");
   const burgersc = join(cFilesFolder, "burgers.c");
+  const main = join(cFilesFolder, "main.c");
+  const all = join(cFilesFolder, "all.h");
+  const errorsh = join(cFilesFolder, "errors.h");
   test("extracts create_burger", () => {
     const symbolsToExtract = new Map<
       string,
@@ -22,7 +29,10 @@ describe("CExtractor", () => {
     });
     const extractedFiles = extractor.extractSymbols(symbolsToExtract);
     expect(extractedFiles.size).toBe(2);
-    const newManifest = generateCDependencyManifest(extractedFiles);
+    const newManifest = generateCDependencyManifest(
+      extractedFiles,
+      dummyLocalConfig,
+    );
     expect(newManifest[burgers]).toBeDefined();
     expect(newManifest[burgersc]).toBeDefined();
     // Expected symbols to be kept
@@ -58,7 +68,10 @@ describe("CExtractor", () => {
     });
     const extractedFiles = extractor.extractSymbols(symbolsToExtract);
     expect(extractedFiles.size).toBe(1);
-    const newManifest = generateCDependencyManifest(extractedFiles);
+    const newManifest = generateCDependencyManifest(
+      extractedFiles,
+      dummyLocalConfig,
+    );
     expect(newManifest[burgers]).toBeDefined();
     // Expected symbols to be kept
     expect(newManifest[burgers].symbols["Drink_t"]).toBeDefined();
@@ -67,4 +80,41 @@ describe("CExtractor", () => {
     // Expected symbols to be removed
     expect(Object.keys(newManifest[burgers].symbols).length).toBe(3);
   });
+
+  test("keeps all.h", () => {
+    const symbolsToExtract = new Map<
+      string,
+      { filePath: string; symbols: Set<string> }
+    >();
+    symbolsToExtract.set(main, {
+      filePath: main,
+      symbols: new Set(["main"]),
+    });
+    const extractedFiles = extractor.extractSymbols(symbolsToExtract);
+    expect(extractedFiles.size).toBe(6);
+    const newManifest = generateCDependencyManifest(
+      extractedFiles,
+      dummyLocalConfig,
+    );
+    expect(newManifest[all]).toBeDefined();
+  });
+
+  test("deletes impossible include", () => {
+    const symbolsToExtract = new Map<
+      string,
+      { filePath: string; symbols: Set<string> }
+    >();
+    symbolsToExtract.set(errorsh, {
+      filePath: errorsh,
+      symbols: new Set(["typedef"]),
+    });
+    const extractedFiles = extractor.extractSymbols(symbolsToExtract);
+    expect(extractedFiles.size).toBe(1);
+    expect(extractedFiles.get(errorsh)).toBeDefined();
+    expect(
+      extractedFiles.get(errorsh)?.content.includes(
+        `#include "thisfiledoesnotexist.h"`,
+      ),
+    ).toBe(false);
+  });
 });

packages/cli/src/languagePlugins/c/extractor/index.ts

@@ -7,15 +7,21 @@ import type { CFile, Symbol } from "../symbolRegistry/types.ts";
 import type { ExportedFile } from "./types.ts";
 import { C_DECLARATION_QUERY } from "../headerResolver/queries.ts";
 import { C_IFDEF_QUERY } from "./queries.ts";
+import { CInvocationResolver } from "../invocationResolver/index.ts";
+import type z from "npm:zod";
+import type { localConfigSchema } from "../../../config/localConfig.ts";
+import { join } from "@std/path";
 
 export class CExtractor {
   manifest: DependencyManifest;
   registry: Map<string, CFile>;
   includeResolver: CIncludeResolver;
+  invocationResolver: CInvocationResolver;
 
   constructor(
     files: Map<string, { path: string; content: string }>,
     manifest: DependencyManifest,
+    napiConfig: z.infer<typeof localConfigSchema>,
   ) {
     this.manifest = manifest;
     const parsedFiles = new Map<
@@ -30,7 +36,13 @@ export class CExtractor {
     }
     const symbolRegistry = new CSymbolRegistry(parsedFiles);
     this.registry = symbolRegistry.getRegistry();
-    this.includeResolver = new CIncludeResolver(symbolRegistry);
+    const outDir = napiConfig.outDir;
+    const includeDirs = napiConfig["c"]?.includedirs ?? [];
+    if (outDir) {
+      includeDirs.push(...includeDirs.map((i) => join(outDir, i)));
+    }
+    this.includeResolver = new CIncludeResolver(symbolRegistry, includeDirs);
+    this.invocationResolver = new CInvocationResolver(this.includeResolver);
   }
 
   /**
@@ -100,7 +112,8 @@ export class CExtractor {
         const ifdefs = C_IFDEF_QUERY.captures(originalFile.rootNode).map((n) =>
           n.node.text
         );
-        const definesToKeep = ifdefs.map((i) => fileInRegistry.symbols.get(i)!);
+        const definesToKeep = ifdefs.map((i) => fileInRegistry.symbols.get(i)!)
+          .filter((i) => i);
         const symbols = new Map<string, Symbol>();
         for (const define of definesToKeep) {
           symbols.set(define.name, define);
@@ -116,6 +129,36 @@ export class CExtractor {
       if (!exportedFile.symbols.has(symbolName)) {
         exportedFile.symbols.set(symbolName, symbol);
       }
+      // Keep the files that recursively lead to a symbol we need
+      const invocations = this.invocationResolver.getInvocationsForSymbol(
+        symbol,
+      );
+      const filestokeep = Array.from(
+        invocations.resolved.values().map((s) =>
+          this.includeResolver.findInclusionChain(
+            symbol.declaration.filepath,
+            s.symbol,
+          )
+        ).filter((c) => c !== undefined),
+      ).flatMap((c) => c.flatMap((f) => this.registry.get(f)!));
+      for (const f of filestokeep) {
+        if (!exportedFiles.has(f.file.path)) {
+          const ifdefs = C_IFDEF_QUERY.captures(f.file.rootNode).map((n) =>
+            n.node.text
+          );
+          const definesToKeep = ifdefs.map((i) => f.symbols.get(i)!)
+            .filter((i) => i);
+          const symbols = new Map<string, Symbol>();
+          for (const define of definesToKeep) {
+            symbols.set(define.name, define);
+          }
+          exportedFiles.set(f.file.path, {
+            symbols,
+            originalFile: f.file,
+            strippedFile: f.file,
+          });
+        }
+      }
     }
     return exportedFiles;
   }
@@ -127,30 +170,100 @@ export class CExtractor {
   #stripFiles(files: Map<string, ExportedFile>) {
     for (const [, file] of files) {
       const rootNode = file.originalFile.rootNode;
-      const matches = C_DECLARATION_QUERY.captures(rootNode);
+      const originalText = rootNode.text; // Original file content
       const symbolsToKeep = new Set(
         file.symbols.values().map((s) => s.declaration.node),
       );
       const symbolsToRemove = new Set<Parser.SyntaxNode>();
+      const matches = C_DECLARATION_QUERY.captures(rootNode);
+
       for (const match of matches) {
         const symbolNode = match.node;
         if (!symbolsToKeep.has(symbolNode)) {
           symbolsToRemove.add(symbolNode);
         }
       }
-      let filetext = rootNode.text;
-      for (const symbolNode of symbolsToRemove) {
-        const symbolText = symbolNode.text;
-        filetext = filetext.replace(symbolText, "");
-      }
-      const strippedFile = cParser.parse(filetext);
+
+      // Helper function to recursively filter nodes
+      const filterNodes = (node: Parser.SyntaxNode): string => {
+        if (symbolsToRemove.has(node)) {
+          return ""; // Skip this node
+        }
+
+        // If the node has children, process them recursively
+        if (["translation_unit", "preproc_ifdef"].includes(node.type)) {
+          let result = "";
+          let lastEndIndex = node.startIndex;
+
+          for (const child of node.children) {
+            // Append the text between the last node and the current child
+            result += originalText.slice(lastEndIndex, child.startIndex);
+            result += filterNodes(child); // Process the child
+            lastEndIndex = child.endIndex;
+          }
+
+          // Append the text after the last child
+          result += originalText.slice(lastEndIndex, node.endIndex);
+          return result;
+        }
+
+        // If the node has no children, return its text
+        return originalText.slice(node.startIndex, node.endIndex);
+      };
+
+      // Rebuild the file content by filtering nodes
+      const newFileContent = filterNodes(rootNode);
+
+      // Compactify the file content
+      const compactedContent = this.#compactifyFile(newFileContent);
+
+      // Parse the new content and update the stripped file
+      const strippedFile = cParser.parse(compactedContent);
       file.strippedFile = {
         path: file.originalFile.path,
         rootNode: strippedFile.rootNode,
       };
     }
   }
 
+  #removeDeletedIncludes(
+    files: Map<string, ExportedFile>,
+  ) {
+    const newproject: Map<
+      string,
+      { path: string; rootNode: Parser.SyntaxNode }
+    > = new Map();
+    for (const [key, value] of files) {
+      newproject.set(key, value.strippedFile);
+    }
+    const newregistry = new CSymbolRegistry(newproject);
+    const newincluderes = new CIncludeResolver(
+      newregistry,
+      this.includeResolver.includeDirs,
+    );
+    newincluderes.getInclusions();
+    for (const [key, value] of files) {
+      const unresolved = newincluderes.unresolvedDirectives.get(key);
+      if (unresolved) {
+        let filetext = value.strippedFile.rootNode.text;
+        for (const path of unresolved) {
+          filetext = filetext.replace(`#include "${path}"`, "");
+        }
+        filetext = this.#compactifyFile(filetext);
+        value.strippedFile.rootNode = cParser.parse(filetext).rootNode;
+      }
+    }
+  }
+
+  #compactifyFile(
+    filetext: string,
+  ): string {
+    // Remove empty lines and useless semicolons
+    filetext = filetext.replace(/^\s*;\s*$/gm, ""); // Remove empty lines with semicolons
+    filetext = filetext.replace(/^\s*[\r\n]+/gm, "\n"); // Remove empty lines
+    return filetext;
+  }
+
   /**
    * Finds the dependencies for a map of symbols.
    * @param symbolsMap - A map of file paths to their corresponding symbols.
@@ -196,6 +309,7 @@ export class CExtractor {
     const symbolsToExtract = this.#findDependenciesForMap(symbolsMap);
     const filesToExport = this.#buildFileMap(symbolsToExtract);
     this.#stripFiles(filesToExport);
+    this.#removeDeletedIncludes(filesToExport);
     const exportedFiles = new Map<string, { path: string; content: string }>();
     for (const [filePath, file] of filesToExport) {
       const content = file.strippedFile.rootNode.text;

packages/cli/src/languagePlugins/c/headerResolver/index.test.ts

@@ -10,6 +10,7 @@ describe("CHeaderResolver", () => {
   const burgers = join(cFilesFolder, "burgers.h");
   const crashcases = join(cFilesFolder, "crashcases.h");
   const errorsh = join(cFilesFolder, "errors.h");
+  const oldmanh = join(cFilesFolder, "oldman.h");
   const file = cFilesMap.get(burgers);
   if (!file) {
     throw new Error(`File not found: ${burgers}`);
@@ -298,4 +299,27 @@ describe("CHeaderResolver", () => {
     const symbolNames = exportedErrorSymbols.map((symbol) => symbol.name);
     expect(symbolNames).toContain("#NAPI_UNNAMED_ENUM_0");
   });
+
+  test("resolves typedefs with same name as associated struct", () => {
+    const oldmanSymbols = resolver.resolveSymbols(
+      cFilesMap.get(oldmanh)!,
+    );
+    expect(oldmanSymbols).toBeDefined();
+    const oldmen = oldmanSymbols.filter((s) => s.name === "OldMan");
+    expect(oldmen).toHaveLength(2);
+    const oldman = oldmen.find((s) => s.type === "typedef");
+    expect(oldman).toBeDefined();
+    if (!oldman) {
+      throw new Error("OldMan is undefined");
+    }
+    expect(oldman.type).toBe("typedef");
+    expect(oldman.specifiers).toEqual([]);
+    expect(oldman.qualifiers).toEqual([]);
+    expect(oldman.node.type).toBe("type_definition");
+    if (!oldman.identifierNode) {
+      throw new Error("oldman.identifierNode is undefined");
+    }
+    expect(oldman.identifierNode.type).toBe("type_identifier");
+    expect(oldman.filepath).toBe(oldmanh);
+  });
 });