summaryrefslogtreecommitdiff
path: root/packages/core/src
diff options
context:
space:
mode:
authormatt korwel <[email protected]>2025-06-07 12:07:58 -0700
committerGitHub <[email protected]>2025-06-07 12:07:58 -0700
commitf1a4e5d4d3f7e4fdcee193622b3e7aebf661a48b (patch)
tree3bd51fcadd75451fbb7b14d558e361122389aac6 /packages/core/src
parent28ff62e7b1b2191c5f5193314523f848a0f3dea5 (diff)
Creating Node AST Tool. (#756)
Diffstat (limited to 'packages/core/src')
-rw-r--r--packages/core/src/config/config.ts2
-rw-r--r--packages/core/src/tools/code_parser.test.ts782
-rw-r--r--packages/core/src/tools/code_parser.ts386
3 files changed, 1170 insertions, 0 deletions
diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts
index c25bc8fc..00b3e35d 100644
--- a/packages/core/src/config/config.ts
+++ b/packages/core/src/config/config.ts
@@ -10,6 +10,7 @@ import * as path from 'node:path';
import process from 'node:process';
import * as os from 'node:os';
import { ToolRegistry } from '../tools/tool-registry.js';
+import { CodeParserTool } from '../tools/code_parser.js'; // Added CodeParserTool
import { LSTool } from '../tools/ls.js';
import { ReadFileTool } from '../tools/read-file.js';
import { GrepTool } from '../tools/grep.js';
@@ -355,6 +356,7 @@ export function createToolRegistry(config: Config): Promise<ToolRegistry> {
registerCoreTool(ShellTool, config);
registerCoreTool(MemoryTool);
registerCoreTool(WebSearchTool, config);
+ registerCoreTool(CodeParserTool, targetDir, config); // Added CodeParserTool
return (async () => {
await registry.discoverTools();
return registry;
diff --git a/packages/core/src/tools/code_parser.test.ts b/packages/core/src/tools/code_parser.test.ts
new file mode 100644
index 00000000..358edc7d
--- /dev/null
+++ b/packages/core/src/tools/code_parser.test.ts
@@ -0,0 +1,782 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {
+ vi,
+ describe,
+ it,
+ expect,
+ beforeEach,
+ afterEach,
+ Mocked,
+} from 'vitest';
+import { CodeParserTool, CodeParserToolParams } from './code_parser.js';
+import { Config } from '../config/config.js';
+import fs from 'fs/promises';
+import { Stats, PathLike } from 'fs'; // Added Stats import
+import path from 'path';
+import os from 'os';
+import actualFs from 'fs'; // For actual fs operations in setup
+
+// Mock fs/promises
+vi.mock('fs/promises');
+
+// Mock tree-sitter and its language grammars
+const mockTreeSitterParse = vi.fn();
+const mockSetLanguage = vi.fn();
+
+vi.mock('tree-sitter', () => ({
+ default: vi.fn().mockImplementation(() => ({
+ setLanguage: mockSetLanguage,
+ parse: mockTreeSitterParse,
+ })),
+}));
+
+const mockPythonGrammar = vi.hoisted(() => ({ name: 'python' }));
+const mockJavaGrammar = vi.hoisted(() => ({ name: 'java' }));
+const mockGoGrammar = vi.hoisted(() => ({ name: 'go' }));
+const mockCSharpGrammar = vi.hoisted(() => ({ name: 'csharp' }));
+const mockTypeScriptGrammar = vi.hoisted(() => ({ name: 'typescript' }));
+const mockTSXGrammar = vi.hoisted(() => ({ name: 'tsx' }));
+const mockRustGrammar = vi.hoisted(() => ({ name: 'rust' })); // Added for Rust
+
+vi.mock('tree-sitter-python', () => ({ default: mockPythonGrammar }));
+vi.mock('tree-sitter-java', () => ({ default: mockJavaGrammar }));
+vi.mock('tree-sitter-go', () => ({ default: mockGoGrammar }));
+vi.mock('tree-sitter-c-sharp', () => ({ default: mockCSharpGrammar }));
+vi.mock('tree-sitter-typescript', () => ({
+ default: {
+ typescript: mockTypeScriptGrammar,
+ tsx: mockTSXGrammar,
+ },
+}));
+vi.mock('tree-sitter-rust', () => ({ default: mockRustGrammar })); // Added for Rust
+
+describe('CodeParserTool', () => {
+ let tempRootDir: string;
+ let tool: CodeParserTool;
+ let mockConfig: Config;
+ const abortSignal = new AbortController().signal;
+
+ // Use Mocked type for fs/promises
+ let mockFs: Mocked<typeof fs>;
+
+ beforeEach(() => {
+ const tempDirPrefix = path.join(os.tmpdir(), 'code-parser-tool-root-');
+ tempRootDir = actualFs.mkdtempSync(tempDirPrefix);
+ tempRootDir = path.resolve(tempRootDir);
+
+ mockConfig = { get: vi.fn() } as unknown as Config;
+ tool = new CodeParserTool(tempRootDir, mockConfig);
+ mockFs = fs as Mocked<typeof fs>;
+
+ mockTreeSitterParse.mockReset();
+ mockSetLanguage.mockReset();
+ mockFs.stat.mockReset();
+ mockFs.readFile.mockReset();
+ mockFs.readdir.mockReset();
+
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(mock_ast)' },
+ });
+ });
+
+ afterEach(() => {
+ if (actualFs.existsSync(tempRootDir)) {
+ actualFs.rmSync(tempRootDir, { recursive: true, force: true });
+ }
+ vi.clearAllMocks();
+ });
+
+ describe('constructor and schema', () => {
+ it('should have correct name', () => {
+ expect(tool.name).toBe('code_parser');
+ });
+
+ it('should have correct schema definition', () => {
+ const schema = tool.schema.parameters!;
+ expect(schema.type).toBe('object');
+ expect(schema.properties).toHaveProperty('path');
+ expect(schema.properties!.path.type).toBe('string');
+ expect(schema.properties!.path.description).toContain('absolute path');
+ expect(schema.properties).toHaveProperty('languages');
+ expect(schema.properties!.languages.type).toBe('array');
+ expect(schema.properties!.languages.description).toContain('go');
+ expect(schema.properties!.languages.description).toContain('csharp');
+ expect(schema.properties!.languages.description).toContain('typescript');
+ expect(schema.properties!.languages.description).toContain('tsx');
+ expect(schema.properties!.languages.description).toContain('javascript');
+ expect(schema.properties!.languages.description).toContain('rust'); // Added for Rust
+ expect(schema.required).toEqual(['path']);
+ });
+ });
+
+ describe('validateToolParams', () => {
+ it('should return null for valid path with languages', () => {
+ const params: CodeParserToolParams = {
+ path: path.join(tempRootDir, 'dir'),
+ languages: [
+ 'python',
+ 'go',
+ 'csharp',
+ 'typescript',
+ 'tsx',
+ 'javascript',
+ ],
+ };
+ expect(tool.validateToolParams(params)).toBeNull();
+ });
+
+ it('should return error for relative path', () => {
+ const params: CodeParserToolParams = { path: 'file.py' };
+ expect(tool.validateToolParams(params)).toMatch(/Path must be absolute/);
+ });
+
+ it('should return error for path outside root directory', () => {
+ const outsidePath = path.resolve(
+ os.tmpdir(),
+ 'some-other-dir',
+ 'file.py',
+ );
+ if (outsidePath.startsWith(tempRootDir)) {
+ console.warn(
+ 'Skipping outside root test due to overlapping temp/outside paths',
+ );
+ return;
+ }
+ const params: CodeParserToolParams = { path: outsidePath };
+ expect(tool.validateToolParams(params)).toMatch(
+ /Path must be within the root directory/,
+ );
+ });
+
+ it('should return error if languages is not an array of strings', () => {
+ const params = {
+ path: path.join(tempRootDir, 'file.py'),
+ languages: [123],
+ } as unknown as CodeParserToolParams;
+ expect(tool.validateToolParams(params)).toBe(
+ 'Languages parameter must be an array of strings.',
+ );
+ });
+ });
+
+ describe('getDescription', () => {
+ it('should return "Parse <shortened_relative_path>"', () => {
+ const filePath = path.join(tempRootDir, 'src', 'app', 'main.py');
+ const params: CodeParserToolParams = { path: filePath };
+ expect(tool.getDescription(params)).toBe('Parse src/app/main.py');
+ });
+ });
+
+ describe('execute', () => {
+ // --- Error Handling Tests ---
+ it('should return validation error if params are invalid', async () => {
+ const params: CodeParserToolParams = { path: 'relative/path.txt' };
+ const result = await tool.execute(params, abortSignal);
+ expect(result.llmContent).toMatch(
+ /Error: Invalid parameters provided. Reason: Path must be absolute/,
+ );
+ expect(result.returnDisplay).toBe('Error: Failed to execute tool.');
+ });
+
+ it('should return error if target path does not exist', async () => {
+ const targetPath = path.join(tempRootDir, 'nonexistent.py');
+ mockFs.stat.mockRejectedValue({
+ code: 'ENOENT',
+ } as NodeJS.ErrnoException);
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+ expect(result.llmContent).toMatch(
+ /Error: Path not found or inaccessible/,
+ );
+ expect(result.returnDisplay).toMatch(
+ /Error: Path not found or inaccessible/,
+ );
+ });
+
+ it('should return error if target path is not a file or directory', async () => {
+ const targetPath = path.join(tempRootDir, 'neither_file_nor_dir');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => false,
+ isDirectory: () => false,
+ size: 0,
+ } as Stats);
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+ expect(result.llmContent).toMatch(
+ /Error: Path is not a file or directory/,
+ );
+ expect(result.returnDisplay).toMatch(
+ /Error: Path is not a file or directory/,
+ );
+ });
+
+ it('should return error if no supported languages are specified or available', async () => {
+ const targetPath = path.join(tempRootDir, 'file.py');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: 100,
+ } as Stats);
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ const originalGetLanguageParser = (tool as any).getLanguageParser;
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (tool as any).getLanguageParser = vi.fn().mockReturnValue(undefined);
+
+ const params: CodeParserToolParams = {
+ path: targetPath,
+ languages: ['fantasy-lang'],
+ };
+ const result = await tool.execute(params, abortSignal);
+ expect(result.llmContent).toMatch(
+ /Error: No supported languages specified for parsing/,
+ );
+ expect(result.returnDisplay).toMatch(
+ /Error: No supported languages to parse/,
+ );
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ (tool as any).getLanguageParser = originalGetLanguageParser; // Restore
+ });
+
+ // --- Single File Parsing Tests ---
+ it('should parse a single Python file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'test.py');
+ const fileContent = 'print("hello")';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(python_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockPythonGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(python_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single Java file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'Test.java');
+ const fileContent = 'class Test {}';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(java_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockJavaGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(java_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single Go file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'main.go');
+ const fileContent = 'package main\nfunc main(){}';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(go_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockGoGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(go_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single C# file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'Program.cs');
+ const fileContent =
+ 'namespace HelloWorld { class Program { static void Main(string[] args) { System.Console.WriteLine("Hello World!"); } } }';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(csharp_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockCSharpGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(csharp_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single TypeScript (.ts) file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'app.ts');
+ const fileContent = 'const x: number = 10;';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(ts_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(ts_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single TSX (.tsx) file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'component.tsx');
+ const fileContent = 'const Comp = () => <div />;';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(tsx_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(tsx_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single JavaScript (.js) file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'script.js');
+ const fileContent = 'console.log("hello");';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(js_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar); // Uses TypeScript grammar for JS
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(js_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a single Rust (.rs) file successfully', async () => {
+ const targetPath = path.join(tempRootDir, 'main.rs');
+ const fileContent = 'fn main() { println!("Hello, Rust!"); }';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(rust_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockRustGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(rust_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should parse a JavaScript JSX (.jsx) file successfully (using tsx parser)', async () => {
+ const targetPath = path.join(tempRootDir, 'component.jsx');
+ const fileContent = 'const Comp = () => <div />;';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockReturnValue({
+ rootNode: { toString: () => '(jsx_ast)' },
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
+ expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
+ expect(result.llmContent).toBe(
+ `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(jsx_ast)\n\n`,
+ );
+ expect(result.returnDisplay).toBe('Parsed 1 file(s).');
+ });
+
+ it('should return error for unsupported file type if specified directly', async () => {
+ const targetPath = path.join(tempRootDir, 'notes.txt');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: 10,
+ } as Stats);
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toMatch(
+ /Error: File .* is not of a supported language type/,
+ );
+ expect(result.returnDisplay).toMatch(
+ /Error: Unsupported file type or language/,
+ );
+ });
+
+ it('should skip file if it exceeds maxFileSize', async () => {
+ const targetPath = path.join(tempRootDir, 'large.py');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: 1024 * 1024 + 1,
+ } as Stats); // 1MB + 1 byte
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(mockFs.readFile).not.toHaveBeenCalled();
+ expect(result.llmContent).toMatch(
+ /Error: Could not parse file .*large.py/,
+ );
+ expect(result.returnDisplay).toBe('Error: Failed to parse file.');
+ });
+
+ it('should return error if parsing a supported file fails internally', async () => {
+ const targetPath = path.join(tempRootDir, 'broken.py');
+ const fileContent = 'print("hello")';
+ mockFs.stat.mockResolvedValue({
+ isFile: () => true,
+ isDirectory: () => false,
+ size: fileContent.length,
+ } as Stats);
+ mockFs.readFile.mockResolvedValue(fileContent);
+ mockTreeSitterParse.mockImplementation(() => {
+ throw new Error('TreeSitterCrashed');
+ });
+
+ const params: CodeParserToolParams = { path: targetPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toMatch(
+ /Error: Could not parse file .*broken.py/,
+ );
+ expect(result.returnDisplay).toMatch(/Error: Failed to parse file./);
+ });
+
+ // --- Directory Parsing Tests ---
+ it('should parse supported files in a directory (including Go, C#, TS, JS, TSX)', async () => {
+ const dirPath = path.join(tempRootDir, 'src');
+ const files = [
+ 'main.py',
+ 'helper.java',
+ 'service.go',
+ 'App.cs',
+ 'logic.ts',
+ 'ui.tsx',
+ 'utils.js',
+ 'main.rs', // Added Rust file
+ 'config.txt',
+ ];
+ const pythonContent = 'import os';
+ const javaContent = 'public class Helper {}';
+ const goContent = 'package main';
+ const csharpContent = 'public class App {}';
+ const tsContent = 'let val: number = 1;';
+ const tsxContent = 'const MyComp = () => <p />;';
+ const jsContent = 'function hello() {}';
+ const rustContent = 'fn start() {}'; // Added Rust content
+
+ mockFs.stat.mockImplementation(async (p) => {
+ if (p === dirPath)
+ return { isFile: () => false, isDirectory: () => true } as Stats;
+ if (p === path.join(dirPath, 'main.py'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: pythonContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'helper.java'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: javaContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'service.go'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: goContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'App.cs'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: csharpContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'logic.ts'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: tsContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'ui.tsx'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: tsxContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'utils.js'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: jsContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'main.rs'))
+ // Added for Rust
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: rustContent.length,
+ } as Stats;
+ if (p === path.join(dirPath, 'config.txt'))
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: 10,
+ } as Stats;
+ throw { code: 'ENOENT' };
+ });
+ mockFs.readdir.mockImplementation(
+ vi.fn(async (p: PathLike): Promise<string[]> => {
+ const dirPath = path.join(tempRootDir, 'src'); // Path for this specific test
+ if (p === dirPath) {
+ return files; // files is in scope for this test
+ }
+ throw new Error(
+ `fs.readdir mock: Unhandled path ${p} in test 'should parse supported files in a directory'`,
+ );
+ }) as unknown as typeof fs.readdir,
+ );
+ mockFs.readFile.mockImplementation(async (p) => {
+ if (p === path.join(dirPath, 'main.py')) return pythonContent;
+ if (p === path.join(dirPath, 'helper.java')) return javaContent;
+ if (p === path.join(dirPath, 'service.go')) return goContent;
+ if (p === path.join(dirPath, 'App.cs')) return csharpContent;
+ if (p === path.join(dirPath, 'logic.ts')) return tsContent;
+ if (p === path.join(dirPath, 'ui.tsx')) return tsxContent;
+ if (p === path.join(dirPath, 'utils.js')) return jsContent;
+ if (p === path.join(dirPath, 'main.rs')) return rustContent; // Added for Rust
+ return '';
+ });
+ mockTreeSitterParse.mockImplementation((content) => {
+ if (content === pythonContent)
+ return { rootNode: { toString: () => '(py_ast_dir)' } };
+ if (content === javaContent)
+ return { rootNode: { toString: () => '(java_ast_dir)' } };
+ if (content === goContent)
+ return { rootNode: { toString: () => '(go_ast_dir)' } };
+ if (content === csharpContent)
+ return { rootNode: { toString: () => '(csharp_ast_dir)' } };
+ if (content === tsContent)
+ return { rootNode: { toString: () => '(ts_ast_dir)' } };
+ if (content === tsxContent)
+ return { rootNode: { toString: () => '(tsx_ast_dir)' } };
+ if (content === jsContent)
+ return { rootNode: { toString: () => '(js_ast_dir)' } };
+ if (content === rustContent)
+ // Added for Rust
+ return { rootNode: { toString: () => '(rust_ast_dir)' } };
+ return { rootNode: { toString: () => '(other_ast)' } };
+ });
+
+ const params: CodeParserToolParams = { path: dirPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'main.py')}-------------\n(py_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'helper.java')}-------------\n(java_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'service.go')}-------------\n(go_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'App.cs')}-------------\n(csharp_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'logic.ts')}-------------\n(ts_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'ui.tsx')}-------------\n(tsx_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ `-------------${path.join(dirPath, 'utils.js')}-------------\n(js_ast_dir)\n`,
+ );
+ expect(result.llmContent).toContain(
+ // Added for Rust
+ `-------------${path.join(dirPath, 'main.rs')}-------------\n(rust_ast_dir)\n`,
+ );
+ expect(result.llmContent).not.toContain('config.txt');
+ expect(result.returnDisplay).toBe('Parsed 8 file(s).'); // Updated count
+ });
+
+ it('should only parse languages specified in the languages parameter for directory', async () => {
+ const dirPath = path.join(tempRootDir, 'mixed_lang_project');
+ const files = [
+ 'script.py',
+ 'Main.java',
+ 'another.py',
+ 'app.go',
+ 'Logic.cs',
+ 'index.ts',
+ 'view.tsx',
+ 'helper.js',
+ 'main.rs', // Added Rust file
+ ];
+ mockFs.stat.mockImplementation(async (p) => {
+ if (p === dirPath)
+ return { isFile: () => false, isDirectory: () => true } as Stats;
+ return {
+ isFile: () => true,
+ isDirectory: () => false,
+ size: 10,
+ } as Stats;
+ });
+ mockFs.readdir.mockImplementation(
+ vi.fn(async (p: PathLike): Promise<string[]> => {
+ // dirPath and files are in scope for this specific test
+ if (p === dirPath) {
+ return files;
+ }
+ throw new Error(
+ `fs.readdir mock: Unhandled path ${p} in test 'should only parse languages specified'`,
+ );
+ }) as unknown as typeof fs.readdir,
+ );
+ mockFs.readFile.mockResolvedValue('content');
+
+ const params: CodeParserToolParams = {
+ path: dirPath,
+ languages: [
+ 'java',
+ 'go',
+ 'csharp',
+ 'typescript',
+ 'tsx',
+ 'javascript',
+ 'rust',
+ ], // Added rust
+ };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toContain(path.join(dirPath, 'Main.java'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'app.go'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'Logic.cs'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'index.ts'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'view.tsx'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'helper.js'));
+ expect(result.llmContent).toContain(path.join(dirPath, 'main.rs')); // Added for Rust
+ expect(result.llmContent).not.toContain('script.py');
+ expect(result.llmContent).not.toContain('another.py');
+ expect(result.returnDisplay).toBe('Parsed 7 file(s).'); // Updated count
+ });
+
+ it('should return "Directory is empty" for an empty directory', async () => {
+ const dirPath = path.join(tempRootDir, 'empty_dir');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => false,
+ isDirectory: () => true,
+ } as Stats);
+ mockFs.readdir.mockResolvedValue([]);
+
+ const params: CodeParserToolParams = { path: dirPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toBe(`Directory ${dirPath} is empty.`);
+ expect(result.returnDisplay).toBe('Directory is empty.');
+ });
+
+ it('should handle error if fs.readdir fails', async () => {
+ const dirPath = path.join(tempRootDir, 'unreadable_dir');
+ mockFs.stat.mockResolvedValue({
+ isFile: () => false,
+ isDirectory: () => true,
+ } as Stats);
+ mockFs.readdir.mockRejectedValue(new Error('Permission denied'));
+
+ const params: CodeParserToolParams = { path: dirPath };
+ const result = await tool.execute(params, abortSignal);
+
+ expect(result.llmContent).toMatch(
+ /Error listing or processing directory/,
+ );
+ expect(result.returnDisplay).toMatch(
+ /Error: Failed to process directory./,
+ );
+ });
+ });
+
+ describe('requiresConfirmation', () => {
+ it('should return null', async () => {
+ const params: CodeParserToolParams = { path: 'anypath' };
+ expect(await tool.requiresConfirmation(params)).toBeNull();
+ });
+ });
+});
diff --git a/packages/core/src/tools/code_parser.ts b/packages/core/src/tools/code_parser.ts
new file mode 100644
index 00000000..12926f29
--- /dev/null
+++ b/packages/core/src/tools/code_parser.ts
@@ -0,0 +1,386 @@
+/**
+ * @license
+ * Copyright 2025 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import Parser from 'tree-sitter';
+import Python from 'tree-sitter-python';
+import Java from 'tree-sitter-java';
+import Go from 'tree-sitter-go';
+import CSharp from 'tree-sitter-c-sharp';
+import TreeSitterTypeScript from 'tree-sitter-typescript';
+import Rust from 'tree-sitter-rust'; // Added
+import fs from 'fs/promises';
+import path from 'path';
+import { BaseTool, ToolResult, ToolCallConfirmationDetails } from './tools.js';
+import { SchemaValidator } from '../utils/schemaValidator.js';
+import { makeRelative, shortenPath } from '../utils/paths.js'; // Removed isWithinRoot
+import { Config } from '../config/config.js';
+
+type TreeSitterLanguage = Parameters<typeof Parser.prototype.setLanguage>[0];
+
+export interface CodeParserToolParams {
+ path: string;
+ ignore?: string[];
+ languages?: string[];
+}
+
+export class CodeParserTool extends BaseTool<CodeParserToolParams, ToolResult> {
+ static readonly Name = 'code_parser';
+
+ private parser: Parser;
+
+ constructor(
+ private rootDirectory: string,
+ private config: Config,
+ ) {
+ super(
+ CodeParserTool.Name,
+ 'CodeParser',
+ 'Parses the code in the specified directory path or a single file to generate AST representations. This should be used to get a better understanding of the codebase when refactoring and building out new features.',
+ {
+ properties: {
+ path: {
+ type: 'string',
+ description:
+ 'The absolute path to the directory or file to parse (must be absolute, not relative)',
+ },
+ languages: {
+ type: 'array',
+ description:
+ 'Optional: specific languages to parse (e.g., ["python", "java", "go", "csharp", "typescript", "tsx", "javascript", "rust"]). Defaults to supported languages.',
+ items: {
+ type: 'string',
+ },
+ },
+ },
+ required: ['path'],
+ type: 'object',
+ },
+ );
+ this.rootDirectory = path.resolve(rootDirectory);
+ this.parser = new Parser();
+ }
+
+ // Added private isWithinRoot method
+ private isWithinRoot(dirpath: string): boolean {
+ const normalizedPath = path.normalize(dirpath);
+ const normalizedRoot = path.normalize(this.rootDirectory);
+ const rootWithSep = normalizedRoot.endsWith(path.sep)
+ ? normalizedRoot
+ : normalizedRoot + path.sep;
+ return (
+ normalizedPath === normalizedRoot ||
+ normalizedPath.startsWith(rootWithSep)
+ );
+ }
+
+ private getLanguageParser(language: string): TreeSitterLanguage | undefined {
+ switch (language.toLowerCase()) {
+ case 'python':
+ return Python;
+ case 'java':
+ return Java;
+ case 'go':
+ return Go;
+ case 'csharp':
+ return CSharp;
+ case 'typescript':
+ return TreeSitterTypeScript.typescript;
+ case 'tsx':
+ return TreeSitterTypeScript.tsx;
+ case 'javascript': // Use TypeScript parser for JS as it handles modern JS well
+ return TreeSitterTypeScript.typescript;
+ case 'rust': // Added
+ return Rust; // Added
+ default:
+ console.warn(
+ `Language '${language}' is not supported by the CodeParserTool.`,
+ );
+ return undefined;
+ }
+ }
+
+ validateToolParams(params: CodeParserToolParams): string | null {
+ if (
+ this.schema.parameters &&
+ !SchemaValidator.validate(
+ this.schema.parameters as Record<string, unknown>,
+ params,
+ )
+ ) {
+ return 'Parameters failed schema validation.';
+ }
+ if (!path.isAbsolute(params.path)) {
+ return `Path must be absolute: ${params.path}`;
+ }
+ if (!this.isWithinRoot(params.path)) {
+ // Use the class method
+ return `Path must be within the root directory (${this.rootDirectory}): ${params.path}`;
+ }
+ if (
+ params.languages &&
+ (!Array.isArray(params.languages) ||
+ !params.languages.every((lang) => typeof lang === 'string'))
+ ) {
+ return 'Languages parameter must be an array of strings.';
+ }
+ return null;
+ }
+
+ getDescription(params: CodeParserToolParams): string {
+ const relativePath = makeRelative(params.path, this.rootDirectory);
+ return `Parse ${shortenPath(relativePath)}`;
+ }
+
+ private errorResult(llmContent: string, returnDisplay: string): ToolResult {
+ return {
+ llmContent,
+ returnDisplay: `Error: ${returnDisplay}`,
+ };
+ }
+
+ private async parseFile(
+ filePath: string,
+ language: string,
+ maxFileSize?: number,
+ ): Promise<string | null> {
+ const langParser = this.getLanguageParser(language);
+ if (!langParser) {
+ return null;
+ }
+ this.parser.setLanguage(langParser);
+
+ try {
+ const stats = await fs.stat(filePath);
+ if (maxFileSize && stats.size > maxFileSize) {
+ console.warn(
+ `File ${filePath} exceeds maxFileSize (${stats.size} > ${maxFileSize}), skipping.`,
+ );
+ return null;
+ }
+
+ const fileContent = await fs.readFile(filePath, 'utf8');
+ const tree = this.parser.parse(fileContent);
+ return this.formatTree(tree.rootNode, 0);
+ } catch (error) {
+ console.error(
+ `Error parsing file ${filePath} with language ${language}:`,
+ error,
+ );
+ return null;
+ }
+ }
+
+ // Helper function to format the AST similar to the Go version
+ private formatTree(node: Parser.SyntaxNode, level: number): string {
+ let formattedTree = '';
+ const indent = ' '.repeat(level);
+ const sexp = node.toString(); // tree-sitter's Node.toString() returns S-expression
+ const maxLength = 100;
+
+ if (sexp.length < maxLength) {
+ // MODIFIED LINE: Removed !sexp.includes('\n')
+ formattedTree += `${indent}${sexp}\n`;
+ return formattedTree;
+ }
+
+ // Expand full format if the S-expression is complex or long
+ formattedTree += `${indent}(${node.type}\n`;
+
+ for (const child of node.namedChildren) {
+ formattedTree += this.formatTree(child, level + 1);
+ }
+
+ // Iterating all children (named and unnamed) to be closer to Go's formatTree.
+ // The original Go code iterates `node.NamedChildCount()` and then `node.ChildCount()`
+ // which implies it processes named children and then all children (including named again).
+ // Here, we iterate named, then iterate all, but skip if already processed as named.
+ // This logic might need further refinement if the exact Go output for unnamed nodes is critical.
+ // For now, focusing on named children as per the Go code's primary loop in formatTree.
+ // If a more exact match for unnamed nodes is needed, the iteration logic for `node.children`
+ // and skipping already processed namedChildren would be added here.
+
+ formattedTree += `${indent})\n`;
+ return formattedTree;
+ }
+
+ private getFileLanguage(filePath: string): string | undefined {
+ const extension = path.extname(filePath).toLowerCase();
+ switch (extension) {
+ case '.py':
+ return 'python';
+ case '.java':
+ return 'java';
+ case '.go':
+ return 'go';
+ case '.cs':
+ return 'csharp';
+ case '.ts':
+ return 'typescript';
+ case '.tsx':
+ return 'tsx';
+ case '.js':
+ return 'javascript';
+ case '.jsx': // Treat jsx as tsx for parsing
+ return 'tsx';
+ case '.mjs':
+ return 'javascript';
+ case '.cjs':
+ return 'javascript';
+ case '.rs': // Added
+ return 'rust'; // Added
+ default:
+ return undefined;
+ }
+ }
+
+ async execute(
+ params: CodeParserToolParams,
+ _signal: AbortSignal,
+ ): Promise<ToolResult> {
+ const validationError = this.validateToolParams(params);
+ if (validationError) {
+ return this.errorResult(
+ `Error: Invalid parameters provided. Reason: ${validationError}`,
+ 'Failed to execute tool.',
+ );
+ }
+
+ const targetPath = params.path;
+ let stats;
+ try {
+ stats = await fs.stat(targetPath);
+ } catch (error) {
+ if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+ return this.errorResult(
+ `Error: Path not found or inaccessible: ${targetPath}`,
+ 'Path not found or inaccessible.',
+ );
+ }
+ return this.errorResult(
+ `Error: Cannot access path: ${(error as Error).message}`,
+ 'Cannot access path.',
+ );
+ }
+
+ const defaultLanguages = [
+ 'python',
+ 'java',
+ 'go',
+ 'csharp',
+ 'typescript',
+ 'tsx',
+ 'javascript',
+ 'rust', // Added
+ ];
+ const languagesToParse = (
+ params.languages && params.languages.length > 0
+ ? params.languages
+ : defaultLanguages
+ ).map((lang) => lang.toLowerCase());
+ const maxFileSize = 1024 * 1024; // 1MB
+
+ const supportedLanguagesToParse = languagesToParse.filter((lang) =>
+ this.getLanguageParser(lang),
+ );
+ if (supportedLanguagesToParse.length === 0) {
+ const availableLangs =
+ defaultLanguages
+ .filter((lang) => this.getLanguageParser(lang))
+ .join(', ') || 'none configured';
+ return this.errorResult(
+ `Error: No supported languages specified for parsing. Requested: ${languagesToParse.join(', ') || 'default'}. Available: ${availableLangs}.`,
+ 'No supported languages to parse.',
+ );
+ }
+
+ let parsedCodeOutput = '';
+ let filesProcessedCount = 0;
+
+ if (stats.isDirectory()) {
+ try {
+ const files = await fs.readdir(targetPath);
+ if (files.length === 0) {
+ return {
+ llmContent: `Directory ${targetPath} is empty.`,
+ returnDisplay: 'Directory is empty.',
+ };
+ }
+
+ for (const file of files) {
+ const filePath = path.join(targetPath, file);
+ let fileStats;
+ try {
+ fileStats = await fs.stat(filePath);
+ } catch {
+ console.warn(`Could not stat file ${filePath}, skipping.`);
+ continue;
+ }
+
+ if (fileStats.isFile()) {
+ const fileLang = this.getFileLanguage(filePath);
+ if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
+ const ast = await this.parseFile(filePath, fileLang, maxFileSize);
+ if (ast) {
+ parsedCodeOutput += `-------------${filePath}-------------\n`;
+ parsedCodeOutput += ast + '\n';
+ filesProcessedCount++;
+ }
+ }
+ }
+ }
+ } catch (error) {
+ return this.errorResult(
+ `Error listing or processing directory ${targetPath}: ${(error as Error).message}`,
+ 'Failed to process directory.',
+ );
+ }
+ } else if (stats.isFile()) {
+ const fileLang = this.getFileLanguage(targetPath);
+ if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
+ const ast = await this.parseFile(targetPath, fileLang, maxFileSize);
+ if (ast) {
+ parsedCodeOutput += `-------------${targetPath}-------------\n`;
+ parsedCodeOutput += ast + '\n';
+ filesProcessedCount++;
+ } else {
+ return this.errorResult(
+ `Error: Could not parse file ${targetPath}. Language '${fileLang}' is supported but parsing failed. Check logs.`,
+ 'Failed to parse file.',
+ );
+ }
+ } else {
+ return this.errorResult(
+ `Error: File ${targetPath} is not of a supported language type for parsing or language not specified. Supported: ${supportedLanguagesToParse.join(', ')}. Detected extension for language: ${fileLang || 'unknown'}.`,
+ 'Unsupported file type or language.',
+ );
+ }
+ } else {
+ return this.errorResult(
+ `Error: Path is not a file or directory: ${targetPath}`,
+ 'Path is not a file or directory.',
+ );
+ }
+
+ if (filesProcessedCount === 0) {
+ return {
+ llmContent: `No files were parsed in ${targetPath}. Ensure files match supported languages (${supportedLanguagesToParse.join(', ')}), are not empty or too large, and are not ignored.`,
+ returnDisplay: 'No files parsed.',
+ };
+ }
+
+ const returnDisplay = `Parsed ${filesProcessedCount} file(s).`;
+ return {
+ llmContent: `Parsed code from ${targetPath}:\n${parsedCodeOutput}`,
+ returnDisplay,
+ };
+ }
+
+ async requiresConfirmation(
+ _params: CodeParserToolParams,
+ ): Promise<ToolCallConfirmationDetails | null> {
+ return null;
+ }
+}