diff --git a/.changeset/open-lines-own.md b/.changeset/open-lines-own.md new file mode 100644 index 00000000..a2492474 --- /dev/null +++ b/.changeset/open-lines-own.md @@ -0,0 +1,5 @@ +--- +"@nodesecure/js-x-ray": minor +--- + +Integrate estree-walker natively using meriyah ESTree types diff --git a/workspaces/js-x-ray/package.json b/workspaces/js-x-ray/package.json index 8ba8cba3..250c4a17 100644 --- a/workspaces/js-x-ray/package.json +++ b/workspaces/js-x-ray/package.json @@ -49,7 +49,6 @@ "@nodesecure/sec-literal": "^1.2.0", "@nodesecure/tracer": "^2.0.0", "digraph-js": "^2.2.3", - "estree-walker": "^3.0.1", "frequency-set": "^1.0.2", "is-minified-code": "^2.0.0", "meriyah": "^6.0.0", diff --git a/workspaces/js-x-ray/src/AstAnalyser.ts b/workspaces/js-x-ray/src/AstAnalyser.ts index e4adc638..e4ccab75 100644 --- a/workspaces/js-x-ray/src/AstAnalyser.ts +++ b/workspaces/js-x-ray/src/AstAnalyser.ts @@ -4,7 +4,6 @@ import fsSync from "node:fs"; import path from "node:path"; // Import Third-party Dependencies -import { walk } from "estree-walker"; import type { ESTree } from "meriyah"; import isMinified from "is-minified-code"; @@ -21,6 +20,7 @@ import { import { isOneLineExpressionExport } from "./utils/index.js"; import { JsSourceParser, type SourceParser } from "./JsSourceParser.js"; import { ProbeRunner, type Probe } from "./ProbeRunner.js"; +import { walkEnter } from "./walker/index.js"; import * as trojan from "./obfuscators/trojan-source.js"; export interface Dependency { @@ -161,19 +161,16 @@ export class AstAnalyser { } // we walk each AST Nodes, this is a purely synchronous I/O - // @ts-expect-error - walk(body, { - enter(node: any) { - // Skip the root of the AST. - if (Array.isArray(node)) { - return; - } - - source.walk(node); - const action = runner.walk(node); - if (action === "skip") { - this.skip(); - } + walkEnter(body, function walk(node) { + // Skip the root of the AST. + if (Array.isArray(node)) { + return; + } + + source.walk(node); + const action = runner.walk(node); + if (action === "skip") { + this.skip(); } }); diff --git a/workspaces/js-x-ray/src/probes/isRequire/RequireCallExpressionWalker.ts b/workspaces/js-x-ray/src/probes/isRequire/RequireCallExpressionWalker.ts index e688d2e9..ab3aee4f 100644 --- a/workspaces/js-x-ray/src/probes/isRequire/RequireCallExpressionWalker.ts +++ b/workspaces/js-x-ray/src/probes/isRequire/RequireCallExpressionWalker.ts @@ -3,7 +3,6 @@ import path from "node:path"; // Import Third-party Dependencies import { Hex } from "@nodesecure/sec-literal"; -import { walk as doWalk } from "estree-walker"; import { arrayExpressionToString, getMemberExpressionIdentifier, @@ -17,6 +16,7 @@ import { isLiteral, isCallExpression } from "../../types/estree.js"; +import { walkEnter } from "../../walker/index.js"; export class RequireCallExpressionWalker { tracer: VariableTracer; @@ -41,47 +41,44 @@ export class RequireCallExpressionWalker { // we need the `this` context of doWalk.enter const self = this; - // @ts-expect-error - doWalk(callExprNode, { - enter(node: any) { - if ( - !isCallExpression(node) || - node.arguments.length === 0 - ) { - return; - } - - const castedNode = node as ESTree.CallExpression; - const rootArgument = castedNode.arguments.at(0)!; - if ( - rootArgument.type === "Literal" && - typeof rootArgument.value === "string" && - Hex.isHex(rootArgument.value) - ) { - self.dependencies.add(Buffer.from(rootArgument.value, "hex").toString()); - this.skip(); - - return; - } - - const fullName = castedNode.callee.type === "MemberExpression" ? - [...getMemberExpressionIdentifier(castedNode.callee)].join(".") : - castedNode.callee.name; - const tracedFullName = self.tracer.getDataFromIdentifier(fullName)?.identifierOrMemberExpr ?? fullName; - switch (tracedFullName) { - case "atob": - self.#handleAtob(castedNode); - break; - case "Buffer.from": - self.#handleBufferFrom(castedNode); - break; - case "require.resolve": - self.#handleRequireResolve(rootArgument); - break; - case "path.join": - self.#handlePathJoin(castedNode); - break; - } + walkEnter(callExprNode, function enter(node) { + if ( + !isCallExpression(node) || + node.arguments.length === 0 + ) { + return; + } + + const castedNode = node as ESTree.CallExpression; + const rootArgument = castedNode.arguments.at(0)!; + if ( + rootArgument.type === "Literal" && + typeof rootArgument.value === "string" && + Hex.isHex(rootArgument.value) + ) { + self.dependencies.add(Buffer.from(rootArgument.value, "hex").toString()); + this.skip(); + + return; + } + + const fullName = castedNode.callee.type === "MemberExpression" ? + [...getMemberExpressionIdentifier(castedNode.callee)].join(".") : + castedNode.callee.name; + const tracedFullName = self.tracer.getDataFromIdentifier(fullName)?.identifierOrMemberExpr ?? fullName; + switch (tracedFullName) { + case "atob": + self.#handleAtob(castedNode); + break; + case "Buffer.from": + self.#handleBufferFrom(castedNode); + break; + case "require.resolve": + self.#handleRequireResolve(rootArgument); + break; + case "path.join": + self.#handlePathJoin(castedNode); + break; } }); diff --git a/workspaces/js-x-ray/src/types/estree.ts b/workspaces/js-x-ray/src/types/estree.ts index 29ebec8c..0d51a245 100644 --- a/workspaces/js-x-ray/src/types/estree.ts +++ b/workspaces/js-x-ray/src/types/estree.ts @@ -10,7 +10,7 @@ export type RegExpLiteral = ESTree.RegExpLiteral & { }; export function isNode( - value: any + value: unknown ): value is ESTree.Node { return ( value !== null && @@ -21,7 +21,7 @@ export function isNode( } export function isLiteral( - node: any + node: unknown ): node is Literal { return isNode(node) && node.type === "Literal" && @@ -29,7 +29,7 @@ export function isLiteral( } export function isTemplateLiteral( - node: any + node: unknown ): node is ESTree.TemplateLiteral { if (!isNode(node) || node.type !== "TemplateLiteral") { return false; @@ -47,7 +47,7 @@ export function isTemplateLiteral( } export function isCallExpression( - node: any + node: unknown ): node is ESTree.CallExpression { return isNode(node) && node.type === "CallExpression"; } diff --git a/workspaces/js-x-ray/src/walker/index.ts b/workspaces/js-x-ray/src/walker/index.ts new file mode 100644 index 00000000..62b5b332 --- /dev/null +++ b/workspaces/js-x-ray/src/walker/index.ts @@ -0,0 +1,26 @@ +// Import Third-party Dependencies +import type { ESTree } from "meriyah"; + +// Import Internal Dependencies +import { SyncWalker, type SyncHandler } from "./walker.sync.js"; + +export type WalkRootNode = ESTree.Program | ESTree.Program["body"] | ESTree.Node; + +export function walk( + ast: WalkRootNode, + { enter, leave }: { enter?: SyncHandler; leave?: SyncHandler; } = {} +) { + const instance = new SyncWalker(enter, leave); + + return instance.visit( + ast as unknown as ESTree.Node, + { parent: null } + ); +} + +export function walkEnter( + ast: WalkRootNode, + enter: SyncHandler +): ESTree.Node | null { + return walk(ast, { enter }); +} diff --git a/workspaces/js-x-ray/src/walker/walker.base.ts b/workspaces/js-x-ray/src/walker/walker.base.ts new file mode 100644 index 00000000..123a434e --- /dev/null +++ b/workspaces/js-x-ray/src/walker/walker.base.ts @@ -0,0 +1,55 @@ +// Import Third-party Dependencies +import type { ESTree } from "meriyah"; + +export interface WalkerContext { + skip: () => void; + remove: () => void; + replace: (node: ESTree.Node) => void; +} + +export class WalkerBase { + should_skip = false; + should_remove = false; + replacement: ESTree.Node | null = null; + context: WalkerContext; + + constructor() { + this.context = { + skip: () => (this.should_skip = true), + remove: () => (this.should_remove = true), + replace: (node) => (this.replacement = node) + }; + } + + // eslint-disable-next-line max-params + replace( + parent: ESTree.Node | null | undefined, + prop: string | number | symbol | null | undefined, + index: number | null | undefined, + node: ESTree.Node + ) { + if (parent && prop) { + if (index === null) { + parent[prop] = node; + } + else { + parent[prop][index] = node; + } + } + } + + remove( + parent: ESTree.Node | null | undefined, + prop: string | number | symbol | null | undefined, + index: number | null | undefined + ) { + if (parent && prop) { + if (index !== null && index !== undefined) { + parent[prop].splice(index, 1); + } + else { + delete parent[prop]; + } + } + } +} diff --git a/workspaces/js-x-ray/src/walker/walker.sync.ts b/workspaces/js-x-ray/src/walker/walker.sync.ts new file mode 100644 index 00000000..f48d971e --- /dev/null +++ b/workspaces/js-x-ray/src/walker/walker.sync.ts @@ -0,0 +1,128 @@ +// Import Third-party Dependencies +import type { ESTree } from "meriyah"; + +// Import Internal Dependencies +import { WalkerBase, type WalkerContext } from "./walker.base.js"; +import { isNode } from "../types/estree.js"; + +export type SyncHandler = ( + this: WalkerContext, + node: ESTree.Node, + context: SyncWalkerVisitorContext +) => void; + +export interface SyncWalkerVisitorContext { + parent: ESTree.Node | null; + prop?: string | number; + index?: number | null; +} + +export class SyncWalker extends WalkerBase { + enter: SyncHandler | undefined; + leave: SyncHandler | undefined; + + constructor( + enter?: SyncHandler, + leave?: SyncHandler + ) { + super(); + this.enter = enter; + this.leave = leave; + } + + visit( + node: ESTree.Node, + options: SyncWalkerVisitorContext + ): ESTree.Node | null { + if (!node) { + return null; + } + + const { parent, prop, index } = options; + let returnedNode = node; + + if (this.enter) { + const _should_skip = this.should_skip; + const _should_remove = this.should_remove; + const _replacement = this.replacement; + this.should_skip = false; + this.should_remove = false; + this.replacement = null; + + this.enter.call(this.context, returnedNode, { parent, prop, index }); + + if (this.replacement) { + returnedNode = this.replacement; + this.replace(parent, prop, index, returnedNode); + } + + if (this.should_remove) { + this.remove(parent, prop, index); + } + + const skipped = this.should_skip; + const removed = this.should_remove; + + this.should_skip = _should_skip; + this.should_remove = _should_remove; + this.replacement = _replacement; + + if (skipped) { + return returnedNode; + } + if (removed) { + return null; + } + } + + for (const key in returnedNode) { + if (!Object.hasOwn(returnedNode, key)) { + continue; + } + const value: unknown = returnedNode[key]; + + if (Array.isArray(value)) { + const nodes: unknown[] = value; + for (let i = 0; i < nodes.length; i++) { + const item = nodes[i]; + const removeItem = isNode(item) && !this.visit(item, { parent: returnedNode, prop: key, index: i }); + if (removeItem) { + i--; + } + } + } + else if (isNode(value)) { + this.visit(value, { parent: returnedNode, prop: key, index: null }); + } + } + + if (this.leave) { + const _replacement = this.replacement; + const _should_remove = this.should_remove; + this.replacement = null; + this.should_remove = false; + + this.leave.call(this.context, returnedNode, { parent, prop, index }); + + if (this.replacement) { + returnedNode = this.replacement; + this.replace(parent, prop, index, returnedNode); + } + + if (this.should_remove) { + this.remove(parent, prop, index); + } + + const removed = this.should_remove; + + this.replacement = _replacement; + this.should_remove = _should_remove; + + if (removed) { + return null; + } + } + + return returnedNode; + } +} diff --git a/workspaces/js-x-ray/test/AstAnalyser.spec.ts b/workspaces/js-x-ray/test/AstAnalyser.spec.ts index c5133ada..0e9c4392 100644 --- a/workspaces/js-x-ray/test/AstAnalyser.spec.ts +++ b/workspaces/js-x-ray/test/AstAnalyser.spec.ts @@ -364,7 +364,9 @@ describe("AstAnalyser", () => { customProbes: [ { name: "name", - initialize: () => calls.push("initialize"), + initialize: () => { + calls.push("initialize"); + }, validateNode: () => [true], main: t.mock.fn(), finalize: () => calls.push("finalize") diff --git a/workspaces/js-x-ray/test/Deobfuscator.spec.ts b/workspaces/js-x-ray/test/Deobfuscator.spec.ts index f385fe22..b5d28e78 100644 --- a/workspaces/js-x-ray/test/Deobfuscator.spec.ts +++ b/workspaces/js-x-ray/test/Deobfuscator.spec.ts @@ -4,11 +4,11 @@ import assert from "node:assert/strict"; // Import Third-party Dependencies import type { ESTree } from "meriyah"; -import { walk } from "estree-walker"; // Import Internal Dependencies import { Deobfuscator } from "../src/Deobfuscator.js"; import { JsSourceParser } from "../src/index.js"; +import { walkEnter } from "../src/walker/index.js"; describe("Deobfuscator", () => { describe("identifiers and counters", () => { @@ -341,14 +341,11 @@ describe("Deobfuscator", () => { function walkAst( body: ESTree.Program["body"], - callback: (node: any) => void = (_node) => undefined + callback: (node: ESTree.Node) => void = (_node) => undefined ) { - // @ts-expect-error - walk(body, { - enter(node) { - if (!Array.isArray(node)) { - callback(node); - } + walkEnter(body, function enter(node) { + if (!Array.isArray(node)) { + callback(node); } }); } diff --git a/workspaces/js-x-ray/test/NodeCounter.spec.ts b/workspaces/js-x-ray/test/NodeCounter.spec.ts index d4ff5496..a6e46df9 100644 --- a/workspaces/js-x-ray/test/NodeCounter.spec.ts +++ b/workspaces/js-x-ray/test/NodeCounter.spec.ts @@ -4,12 +4,12 @@ import assert from "node:assert/strict"; // Import Third-party Dependencies import type { ESTree } from "meriyah"; -import { walk } from "estree-walker"; // Import Internal Dependencies import { NodeCounter } from "../src/NodeCounter.js"; import { JsSourceParser } from "../src/index.js"; import { isNode } from "../src/types/estree.js"; +import { walkEnter } from "../src/walker/index.js"; describe("NodeCounter", () => { describe("constructor", () => { @@ -130,14 +130,12 @@ describe("NodeCounter", () => { }); function walkAst( - body: any, - callback: (node: any) => void = () => void 0 + body: ESTree.Program["body"], + callback: (node: ESTree.Node) => void = () => void 0 ) { - walk(body, { - enter(node) { - if (!Array.isArray(node)) { - callback(node); - } + walkEnter(body, function enter(node) { + if (!Array.isArray(node)) { + callback(node); } }); } diff --git a/workspaces/js-x-ray/test/utils/index.ts b/workspaces/js-x-ray/test/utils/index.ts index a5fc49d3..c1c02b92 100644 --- a/workspaces/js-x-ray/test/utils/index.ts +++ b/workspaces/js-x-ray/test/utils/index.ts @@ -1,6 +1,5 @@ // Import Third-party Dependencies import * as meriyah from "meriyah"; -import { walk } from "estree-walker"; // Import Internal Dependencies import { @@ -12,6 +11,7 @@ import { ProbeRunner, type Probe } from "../../src/ProbeRunner.js"; +import { walk } from "../../src/walker/index.js"; export function getWarningKind( warnings: Warning[] @@ -52,7 +52,7 @@ export function getSastAnalysis( const self = this; walk(body, { - enter(node: any) { + enter(node: meriyah.ESTree.Node) { // Skip the root of the AST. if (Array.isArray(node)) { return; diff --git a/workspaces/js-x-ray/test/walker.spec.ts b/workspaces/js-x-ray/test/walker.spec.ts new file mode 100644 index 00000000..e0a974e4 --- /dev/null +++ b/workspaces/js-x-ray/test/walker.spec.ts @@ -0,0 +1,391 @@ +// Import Node.js Dependencies +import { describe, it } from "node:test"; +import assert from "node:assert"; + +// Import Third-party Dependencies +import type { ESTree } from "meriyah"; + +// Import Internal Dependencies +import { walk } from "../src/walker/index.js"; + +describe("walk", () => { + it("walks a malformed node", () => { + const block: any = [ + { + type: "Foo", + answer: undefined + }, + { + type: "Foo", + answer: { + type: "Answer", + value: 42 + } + } + ]; + + let answer: any; + walk( + // @ts-expect-error + { type: "Test", block }, + { + enter(node: any) { + if (node.type === "Answer") { + answer = node; + } + } + } + ); + + assert.strictEqual(answer, block[1].answer); + }); + + it("walks an AST", () => { + const ast: ESTree.Program = { + type: "Program", + body: [ + { + type: "VariableDeclaration", + declarations: [ + { + type: "VariableDeclarator", + id: { type: "Identifier", name: "a" }, + init: { type: "Literal", value: 1, raw: "1" } + }, + { + type: "VariableDeclarator", + id: { type: "Identifier", name: "b" }, + init: { type: "Literal", value: 2, raw: "2" } + } + ], + kind: "var" + } + ], + sourceType: "module" + }; + + const entered: ESTree.Node[] = []; + const left: ESTree.Node[] = []; + + walk(ast, { + enter(node) { + entered.push(node); + }, + leave(node) { + left.push(node); + } + }); + + // @ts-expect-error + const declarations = ast.body[0].declarations; + + assert.deepEqual(entered, [ + ast, + ast.body[0], + declarations[0], + declarations[0].id, + declarations[0].init, + declarations[1], + declarations[1].id, + declarations[1].init + ]); + + assert.deepEqual(left, [ + declarations[0].id, + declarations[0].init, + declarations[0], + declarations[1].id, + declarations[1].init, + declarations[1], + ast.body[0], + ast + ]); + }); + + it("handles null literals", () => { + const ast: ESTree.Program = { + type: "Program", + start: 0, + end: 8, + body: [ + { + type: "ExpressionStatement", + start: 0, + end: 5, + expression: { + type: "Literal", + start: 0, + end: 4, + value: null, + raw: "null" + } + }, + { + type: "ExpressionStatement", + start: 6, + end: 8, + expression: { + type: "Literal", + start: 6, + end: 7, + value: 1, + raw: "1" + } + } + ], + sourceType: "module" + }; + + walk(ast, { + enter() { + // do nothing + }, + leave() { + // do nothing + } + }); + + assert.ok(true); + }); + + it("allows walk() to be invoked within a walk, without context corruption", () => { + const ast: ESTree.Program = { + type: "Program", + start: 0, + end: 8, + body: [ + { + type: "ExpressionStatement", + start: 0, + end: 6, + expression: { + type: "BinaryExpression", + start: 0, + end: 5, + left: { + type: "Identifier", + start: 0, + end: 1, + name: "a" + }, + operator: "+", + right: { + type: "Identifier", + start: 4, + end: 5, + name: "b" + } + } + } + ], + sourceType: "module" + }; + + const identifiers: string[] = []; + + walk(ast, { + enter(node) { + if (node.type === "ExpressionStatement") { + walk(node, { + enter() { + this.skip(); + } + }); + } + + if (node.type === "Identifier") { + identifiers.push(node.name); + } + } + }); + + assert.deepEqual(identifiers, ["a", "b"]); + }); + + it("replaces a node", () => { + const phases = ["enter", "leave"] as const; + for (const phase of phases) { + const ast: ESTree.Program = { + type: "Program", + start: 0, + end: 8, + body: [ + { + type: "ExpressionStatement", + start: 0, + end: 6, + expression: { + type: "BinaryExpression", + start: 0, + end: 5, + left: { + type: "Identifier", + start: 0, + end: 1, + name: "a" + }, + operator: "+", + right: { + type: "Identifier", + start: 4, + end: 5, + name: "b" + } + } + } + ], + sourceType: "module" + }; + + const forty_two: ESTree.Literal = { + type: "Literal", + value: 42, + raw: "42" + }; + + walk(ast, { + [phase](node: ESTree.Node) { + if (node.type === "Identifier" && node.name === "b") { + this.replace(forty_two); + } + } + }); + + // @ts-expect-error + assert.strictEqual(ast.body[0].expression.right, forty_two); + } + }); + + it("replaces a top-level node", () => { + const ast: ESTree.Identifier = { + type: "Identifier", + name: "answer" + }; + + const forty_two: ESTree.Literal = { + type: "Literal", + value: 42, + raw: "42" + }; + + const node = walk(ast, { + enter(node) { + if (node.type === "Identifier" && node.name === "answer") { + this.replace(forty_two); + } + } + }); + + assert.strictEqual(node, forty_two); + }); + + it("removes a node property", () => { + const phases = ["enter", "leave"] as const; + for (const phase of phases) { + const ast: ESTree.Program = { + type: "Program", + start: 0, + end: 8, + body: [ + { + type: "ExpressionStatement", + start: 0, + end: 6, + expression: { + type: "BinaryExpression", + start: 0, + end: 5, + left: { + type: "Identifier", + start: 0, + end: 1, + name: "a" + }, + operator: "+", + right: { + type: "Identifier", + start: 4, + end: 5, + name: "b" + } + } + } + ], + sourceType: "module" + }; + + walk(ast, { + [phase](node: ESTree.Node) { + if (node.type === "Identifier" && node.name === "b") { + this.remove(); + } + } + }); + + // @ts-expect-error + assert.strictEqual(ast.body[0].expression.right, undefined); + } + }); + + it("removes a node from array", () => { + const phases = ["enter", "leave"] as const; + for (const phase of phases) { + const ast: ESTree.Program = { + type: "Program", + body: [ + { + type: "VariableDeclaration", + declarations: [ + { + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "a" + }, + init: null + }, + { + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "b" + }, + init: null + }, + { + type: "VariableDeclarator", + id: { + type: "Identifier", + name: "c" + }, + init: null + } + ], + kind: "let" + } + ], + sourceType: "module" + }; + + const visitedIndex: number[] = []; + walk(ast, { + [phase](node, ctx) { + if (node.type === "VariableDeclarator") { + visitedIndex.push(ctx.index); + if (node.id.name === "a" || node.id.name === "b") { + this.remove(); + } + } + } + }); + + // @ts-expect-error + const declarations = ast.body[0].declarations; + + assert.strictEqual(declarations.length, 1); + assert.strictEqual(declarations[0].id.name, "c"); + assert.strictEqual(visitedIndex.length, 3); + assert.deepEqual(visitedIndex, [0, 0, 0]); + } + }); +}); diff --git a/workspaces/tracer/package.json b/workspaces/tracer/package.json index 1a92e675..2fc2ca31 100644 --- a/workspaces/tracer/package.json +++ b/workspaces/tracer/package.json @@ -29,5 +29,8 @@ "@nodesecure/estree-ast-utils": "^4.0.0", "meriyah": "^6.1.3", "ts-pattern": "^5.7.1" + }, + "devDependencies": { + "estree-walker": "^3.0.3" } }