summaryrefslogtreecommitdiff
path: root/packages/core/src
diff options
context:
space:
mode:
authormatt korwel <[email protected]>2025-06-08 19:07:25 -0700
committerGitHub <[email protected]>2025-06-08 19:07:25 -0700
commit37edbd8c18c19d28c290f6dc2c5d54add0144479 (patch)
treee044d0c06c53799b005c3fa6190ed109c6033103 /packages/core/src
parentccdd1df03935163d5fa39a36873a50c33c17b3a6 (diff)
Rollforward AST changes to unblock Sandboxing (#863)
Diffstat (limited to 'packages/core/src')
-rw-r--r--packages/core/src/config/config.ts2
-rw-r--r--packages/core/src/tools/code_parser.test.ts782
-rw-r--r--packages/core/src/tools/code_parser.ts386
3 files changed, 0 insertions, 1170 deletions
diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts
index 4c8fad65..80446848 100644
--- a/packages/core/src/config/config.ts
+++ b/packages/core/src/config/config.ts
@@ -11,7 +11,6 @@ import process from 'node:process';
import * as os from 'node:os';
import { ContentGeneratorConfig } from '../core/contentGenerator.js';
import { ToolRegistry } from '../tools/tool-registry.js';
-import { CodeParserTool } from '../tools/code_parser.js'; // Added CodeParserTool
import { LSTool } from '../tools/ls.js';
import { ReadFileTool } from '../tools/read-file.js';
import { GrepTool } from '../tools/grep.js';
@@ -350,7 +349,6 @@ export function createToolRegistry(config: Config): Promise<ToolRegistry> {
registerCoreTool(ShellTool, config);
registerCoreTool(MemoryTool);
registerCoreTool(WebSearchTool, config);
- registerCoreTool(CodeParserTool, targetDir, config); // Added CodeParserTool
return (async () => {
await registry.discoverTools();
return registry;
diff --git a/packages/core/src/tools/code_parser.test.ts b/packages/core/src/tools/code_parser.test.ts
deleted file mode 100644
index 358edc7d..00000000
--- a/packages/core/src/tools/code_parser.test.ts
+++ /dev/null
@@ -1,782 +0,0 @@
-/**
- * @license
- * Copyright 2025 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-
-import {
- vi,
- describe,
- it,
- expect,
- beforeEach,
- afterEach,
- Mocked,
-} from 'vitest';
-import { CodeParserTool, CodeParserToolParams } from './code_parser.js';
-import { Config } from '../config/config.js';
-import fs from 'fs/promises';
-import { Stats, PathLike } from 'fs'; // Added Stats import
-import path from 'path';
-import os from 'os';
-import actualFs from 'fs'; // For actual fs operations in setup
-
-// Mock fs/promises
-vi.mock('fs/promises');
-
-// Mock tree-sitter and its language grammars
-const mockTreeSitterParse = vi.fn();
-const mockSetLanguage = vi.fn();
-
-vi.mock('tree-sitter', () => ({
- default: vi.fn().mockImplementation(() => ({
- setLanguage: mockSetLanguage,
- parse: mockTreeSitterParse,
- })),
-}));
-
-const mockPythonGrammar = vi.hoisted(() => ({ name: 'python' }));
-const mockJavaGrammar = vi.hoisted(() => ({ name: 'java' }));
-const mockGoGrammar = vi.hoisted(() => ({ name: 'go' }));
-const mockCSharpGrammar = vi.hoisted(() => ({ name: 'csharp' }));
-const mockTypeScriptGrammar = vi.hoisted(() => ({ name: 'typescript' }));
-const mockTSXGrammar = vi.hoisted(() => ({ name: 'tsx' }));
-const mockRustGrammar = vi.hoisted(() => ({ name: 'rust' })); // Added for Rust
-
-vi.mock('tree-sitter-python', () => ({ default: mockPythonGrammar }));
-vi.mock('tree-sitter-java', () => ({ default: mockJavaGrammar }));
-vi.mock('tree-sitter-go', () => ({ default: mockGoGrammar }));
-vi.mock('tree-sitter-c-sharp', () => ({ default: mockCSharpGrammar }));
-vi.mock('tree-sitter-typescript', () => ({
- default: {
- typescript: mockTypeScriptGrammar,
- tsx: mockTSXGrammar,
- },
-}));
-vi.mock('tree-sitter-rust', () => ({ default: mockRustGrammar })); // Added for Rust
-
-describe('CodeParserTool', () => {
- let tempRootDir: string;
- let tool: CodeParserTool;
- let mockConfig: Config;
- const abortSignal = new AbortController().signal;
-
- // Use Mocked type for fs/promises
- let mockFs: Mocked<typeof fs>;
-
- beforeEach(() => {
- const tempDirPrefix = path.join(os.tmpdir(), 'code-parser-tool-root-');
- tempRootDir = actualFs.mkdtempSync(tempDirPrefix);
- tempRootDir = path.resolve(tempRootDir);
-
- mockConfig = { get: vi.fn() } as unknown as Config;
- tool = new CodeParserTool(tempRootDir, mockConfig);
- mockFs = fs as Mocked<typeof fs>;
-
- mockTreeSitterParse.mockReset();
- mockSetLanguage.mockReset();
- mockFs.stat.mockReset();
- mockFs.readFile.mockReset();
- mockFs.readdir.mockReset();
-
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(mock_ast)' },
- });
- });
-
- afterEach(() => {
- if (actualFs.existsSync(tempRootDir)) {
- actualFs.rmSync(tempRootDir, { recursive: true, force: true });
- }
- vi.clearAllMocks();
- });
-
- describe('constructor and schema', () => {
- it('should have correct name', () => {
- expect(tool.name).toBe('code_parser');
- });
-
- it('should have correct schema definition', () => {
- const schema = tool.schema.parameters!;
- expect(schema.type).toBe('object');
- expect(schema.properties).toHaveProperty('path');
- expect(schema.properties!.path.type).toBe('string');
- expect(schema.properties!.path.description).toContain('absolute path');
- expect(schema.properties).toHaveProperty('languages');
- expect(schema.properties!.languages.type).toBe('array');
- expect(schema.properties!.languages.description).toContain('go');
- expect(schema.properties!.languages.description).toContain('csharp');
- expect(schema.properties!.languages.description).toContain('typescript');
- expect(schema.properties!.languages.description).toContain('tsx');
- expect(schema.properties!.languages.description).toContain('javascript');
- expect(schema.properties!.languages.description).toContain('rust'); // Added for Rust
- expect(schema.required).toEqual(['path']);
- });
- });
-
- describe('validateToolParams', () => {
- it('should return null for valid path with languages', () => {
- const params: CodeParserToolParams = {
- path: path.join(tempRootDir, 'dir'),
- languages: [
- 'python',
- 'go',
- 'csharp',
- 'typescript',
- 'tsx',
- 'javascript',
- ],
- };
- expect(tool.validateToolParams(params)).toBeNull();
- });
-
- it('should return error for relative path', () => {
- const params: CodeParserToolParams = { path: 'file.py' };
- expect(tool.validateToolParams(params)).toMatch(/Path must be absolute/);
- });
-
- it('should return error for path outside root directory', () => {
- const outsidePath = path.resolve(
- os.tmpdir(),
- 'some-other-dir',
- 'file.py',
- );
- if (outsidePath.startsWith(tempRootDir)) {
- console.warn(
- 'Skipping outside root test due to overlapping temp/outside paths',
- );
- return;
- }
- const params: CodeParserToolParams = { path: outsidePath };
- expect(tool.validateToolParams(params)).toMatch(
- /Path must be within the root directory/,
- );
- });
-
- it('should return error if languages is not an array of strings', () => {
- const params = {
- path: path.join(tempRootDir, 'file.py'),
- languages: [123],
- } as unknown as CodeParserToolParams;
- expect(tool.validateToolParams(params)).toBe(
- 'Languages parameter must be an array of strings.',
- );
- });
- });
-
- describe('getDescription', () => {
- it('should return "Parse <shortened_relative_path>"', () => {
- const filePath = path.join(tempRootDir, 'src', 'app', 'main.py');
- const params: CodeParserToolParams = { path: filePath };
- expect(tool.getDescription(params)).toBe('Parse src/app/main.py');
- });
- });
-
- describe('execute', () => {
- // --- Error Handling Tests ---
- it('should return validation error if params are invalid', async () => {
- const params: CodeParserToolParams = { path: 'relative/path.txt' };
- const result = await tool.execute(params, abortSignal);
- expect(result.llmContent).toMatch(
- /Error: Invalid parameters provided. Reason: Path must be absolute/,
- );
- expect(result.returnDisplay).toBe('Error: Failed to execute tool.');
- });
-
- it('should return error if target path does not exist', async () => {
- const targetPath = path.join(tempRootDir, 'nonexistent.py');
- mockFs.stat.mockRejectedValue({
- code: 'ENOENT',
- } as NodeJS.ErrnoException);
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
- expect(result.llmContent).toMatch(
- /Error: Path not found or inaccessible/,
- );
- expect(result.returnDisplay).toMatch(
- /Error: Path not found or inaccessible/,
- );
- });
-
- it('should return error if target path is not a file or directory', async () => {
- const targetPath = path.join(tempRootDir, 'neither_file_nor_dir');
- mockFs.stat.mockResolvedValue({
- isFile: () => false,
- isDirectory: () => false,
- size: 0,
- } as Stats);
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
- expect(result.llmContent).toMatch(
- /Error: Path is not a file or directory/,
- );
- expect(result.returnDisplay).toMatch(
- /Error: Path is not a file or directory/,
- );
- });
-
- it('should return error if no supported languages are specified or available', async () => {
- const targetPath = path.join(tempRootDir, 'file.py');
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: 100,
- } as Stats);
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- const originalGetLanguageParser = (tool as any).getLanguageParser;
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- (tool as any).getLanguageParser = vi.fn().mockReturnValue(undefined);
-
- const params: CodeParserToolParams = {
- path: targetPath,
- languages: ['fantasy-lang'],
- };
- const result = await tool.execute(params, abortSignal);
- expect(result.llmContent).toMatch(
- /Error: No supported languages specified for parsing/,
- );
- expect(result.returnDisplay).toMatch(
- /Error: No supported languages to parse/,
- );
- // eslint-disable-next-line @typescript-eslint/no-explicit-any
- (tool as any).getLanguageParser = originalGetLanguageParser; // Restore
- });
-
- // --- Single File Parsing Tests ---
- it('should parse a single Python file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'test.py');
- const fileContent = 'print("hello")';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(python_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockPythonGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(python_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single Java file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'Test.java');
- const fileContent = 'class Test {}';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(java_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockJavaGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(java_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single Go file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'main.go');
- const fileContent = 'package main\nfunc main(){}';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(go_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockGoGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(go_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single C# file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'Program.cs');
- const fileContent =
- 'namespace HelloWorld { class Program { static void Main(string[] args) { System.Console.WriteLine("Hello World!"); } } }';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(csharp_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockCSharpGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(csharp_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single TypeScript (.ts) file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'app.ts');
- const fileContent = 'const x: number = 10;';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(ts_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(ts_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single TSX (.tsx) file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'component.tsx');
- const fileContent = 'const Comp = () => <div />;';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(tsx_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(tsx_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single JavaScript (.js) file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'script.js');
- const fileContent = 'console.log("hello");';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(js_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockTypeScriptGrammar); // Uses TypeScript grammar for JS
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(js_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a single Rust (.rs) file successfully', async () => {
- const targetPath = path.join(tempRootDir, 'main.rs');
- const fileContent = 'fn main() { println!("Hello, Rust!"); }';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(rust_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockRustGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(rust_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should parse a JavaScript JSX (.jsx) file successfully (using tsx parser)', async () => {
- const targetPath = path.join(tempRootDir, 'component.jsx');
- const fileContent = 'const Comp = () => <div />;';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockReturnValue({
- rootNode: { toString: () => '(jsx_ast)' },
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockSetLanguage).toHaveBeenCalledWith(mockTSXGrammar);
- expect(mockTreeSitterParse).toHaveBeenCalledWith(fileContent);
- expect(result.llmContent).toBe(
- `Parsed code from ${targetPath}:\n-------------${targetPath}-------------\n(jsx_ast)\n\n`,
- );
- expect(result.returnDisplay).toBe('Parsed 1 file(s).');
- });
-
- it('should return error for unsupported file type if specified directly', async () => {
- const targetPath = path.join(tempRootDir, 'notes.txt');
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: 10,
- } as Stats);
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toMatch(
- /Error: File .* is not of a supported language type/,
- );
- expect(result.returnDisplay).toMatch(
- /Error: Unsupported file type or language/,
- );
- });
-
- it('should skip file if it exceeds maxFileSize', async () => {
- const targetPath = path.join(tempRootDir, 'large.py');
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: 1024 * 1024 + 1,
- } as Stats); // 1MB + 1 byte
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(mockFs.readFile).not.toHaveBeenCalled();
- expect(result.llmContent).toMatch(
- /Error: Could not parse file .*large.py/,
- );
- expect(result.returnDisplay).toBe('Error: Failed to parse file.');
- });
-
- it('should return error if parsing a supported file fails internally', async () => {
- const targetPath = path.join(tempRootDir, 'broken.py');
- const fileContent = 'print("hello")';
- mockFs.stat.mockResolvedValue({
- isFile: () => true,
- isDirectory: () => false,
- size: fileContent.length,
- } as Stats);
- mockFs.readFile.mockResolvedValue(fileContent);
- mockTreeSitterParse.mockImplementation(() => {
- throw new Error('TreeSitterCrashed');
- });
-
- const params: CodeParserToolParams = { path: targetPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toMatch(
- /Error: Could not parse file .*broken.py/,
- );
- expect(result.returnDisplay).toMatch(/Error: Failed to parse file./);
- });
-
- // --- Directory Parsing Tests ---
- it('should parse supported files in a directory (including Go, C#, TS, JS, TSX)', async () => {
- const dirPath = path.join(tempRootDir, 'src');
- const files = [
- 'main.py',
- 'helper.java',
- 'service.go',
- 'App.cs',
- 'logic.ts',
- 'ui.tsx',
- 'utils.js',
- 'main.rs', // Added Rust file
- 'config.txt',
- ];
- const pythonContent = 'import os';
- const javaContent = 'public class Helper {}';
- const goContent = 'package main';
- const csharpContent = 'public class App {}';
- const tsContent = 'let val: number = 1;';
- const tsxContent = 'const MyComp = () => <p />;';
- const jsContent = 'function hello() {}';
- const rustContent = 'fn start() {}'; // Added Rust content
-
- mockFs.stat.mockImplementation(async (p) => {
- if (p === dirPath)
- return { isFile: () => false, isDirectory: () => true } as Stats;
- if (p === path.join(dirPath, 'main.py'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: pythonContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'helper.java'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: javaContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'service.go'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: goContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'App.cs'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: csharpContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'logic.ts'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: tsContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'ui.tsx'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: tsxContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'utils.js'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: jsContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'main.rs'))
- // Added for Rust
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: rustContent.length,
- } as Stats;
- if (p === path.join(dirPath, 'config.txt'))
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: 10,
- } as Stats;
- throw { code: 'ENOENT' };
- });
- mockFs.readdir.mockImplementation(
- vi.fn(async (p: PathLike): Promise<string[]> => {
- const dirPath = path.join(tempRootDir, 'src'); // Path for this specific test
- if (p === dirPath) {
- return files; // files is in scope for this test
- }
- throw new Error(
- `fs.readdir mock: Unhandled path ${p} in test 'should parse supported files in a directory'`,
- );
- }) as unknown as typeof fs.readdir,
- );
- mockFs.readFile.mockImplementation(async (p) => {
- if (p === path.join(dirPath, 'main.py')) return pythonContent;
- if (p === path.join(dirPath, 'helper.java')) return javaContent;
- if (p === path.join(dirPath, 'service.go')) return goContent;
- if (p === path.join(dirPath, 'App.cs')) return csharpContent;
- if (p === path.join(dirPath, 'logic.ts')) return tsContent;
- if (p === path.join(dirPath, 'ui.tsx')) return tsxContent;
- if (p === path.join(dirPath, 'utils.js')) return jsContent;
- if (p === path.join(dirPath, 'main.rs')) return rustContent; // Added for Rust
- return '';
- });
- mockTreeSitterParse.mockImplementation((content) => {
- if (content === pythonContent)
- return { rootNode: { toString: () => '(py_ast_dir)' } };
- if (content === javaContent)
- return { rootNode: { toString: () => '(java_ast_dir)' } };
- if (content === goContent)
- return { rootNode: { toString: () => '(go_ast_dir)' } };
- if (content === csharpContent)
- return { rootNode: { toString: () => '(csharp_ast_dir)' } };
- if (content === tsContent)
- return { rootNode: { toString: () => '(ts_ast_dir)' } };
- if (content === tsxContent)
- return { rootNode: { toString: () => '(tsx_ast_dir)' } };
- if (content === jsContent)
- return { rootNode: { toString: () => '(js_ast_dir)' } };
- if (content === rustContent)
- // Added for Rust
- return { rootNode: { toString: () => '(rust_ast_dir)' } };
- return { rootNode: { toString: () => '(other_ast)' } };
- });
-
- const params: CodeParserToolParams = { path: dirPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'main.py')}-------------\n(py_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'helper.java')}-------------\n(java_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'service.go')}-------------\n(go_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'App.cs')}-------------\n(csharp_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'logic.ts')}-------------\n(ts_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'ui.tsx')}-------------\n(tsx_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- `-------------${path.join(dirPath, 'utils.js')}-------------\n(js_ast_dir)\n`,
- );
- expect(result.llmContent).toContain(
- // Added for Rust
- `-------------${path.join(dirPath, 'main.rs')}-------------\n(rust_ast_dir)\n`,
- );
- expect(result.llmContent).not.toContain('config.txt');
- expect(result.returnDisplay).toBe('Parsed 8 file(s).'); // Updated count
- });
-
- it('should only parse languages specified in the languages parameter for directory', async () => {
- const dirPath = path.join(tempRootDir, 'mixed_lang_project');
- const files = [
- 'script.py',
- 'Main.java',
- 'another.py',
- 'app.go',
- 'Logic.cs',
- 'index.ts',
- 'view.tsx',
- 'helper.js',
- 'main.rs', // Added Rust file
- ];
- mockFs.stat.mockImplementation(async (p) => {
- if (p === dirPath)
- return { isFile: () => false, isDirectory: () => true } as Stats;
- return {
- isFile: () => true,
- isDirectory: () => false,
- size: 10,
- } as Stats;
- });
- mockFs.readdir.mockImplementation(
- vi.fn(async (p: PathLike): Promise<string[]> => {
- // dirPath and files are in scope for this specific test
- if (p === dirPath) {
- return files;
- }
- throw new Error(
- `fs.readdir mock: Unhandled path ${p} in test 'should only parse languages specified'`,
- );
- }) as unknown as typeof fs.readdir,
- );
- mockFs.readFile.mockResolvedValue('content');
-
- const params: CodeParserToolParams = {
- path: dirPath,
- languages: [
- 'java',
- 'go',
- 'csharp',
- 'typescript',
- 'tsx',
- 'javascript',
- 'rust',
- ], // Added rust
- };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toContain(path.join(dirPath, 'Main.java'));
- expect(result.llmContent).toContain(path.join(dirPath, 'app.go'));
- expect(result.llmContent).toContain(path.join(dirPath, 'Logic.cs'));
- expect(result.llmContent).toContain(path.join(dirPath, 'index.ts'));
- expect(result.llmContent).toContain(path.join(dirPath, 'view.tsx'));
- expect(result.llmContent).toContain(path.join(dirPath, 'helper.js'));
- expect(result.llmContent).toContain(path.join(dirPath, 'main.rs')); // Added for Rust
- expect(result.llmContent).not.toContain('script.py');
- expect(result.llmContent).not.toContain('another.py');
- expect(result.returnDisplay).toBe('Parsed 7 file(s).'); // Updated count
- });
-
- it('should return "Directory is empty" for an empty directory', async () => {
- const dirPath = path.join(tempRootDir, 'empty_dir');
- mockFs.stat.mockResolvedValue({
- isFile: () => false,
- isDirectory: () => true,
- } as Stats);
- mockFs.readdir.mockResolvedValue([]);
-
- const params: CodeParserToolParams = { path: dirPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toBe(`Directory ${dirPath} is empty.`);
- expect(result.returnDisplay).toBe('Directory is empty.');
- });
-
- it('should handle error if fs.readdir fails', async () => {
- const dirPath = path.join(tempRootDir, 'unreadable_dir');
- mockFs.stat.mockResolvedValue({
- isFile: () => false,
- isDirectory: () => true,
- } as Stats);
- mockFs.readdir.mockRejectedValue(new Error('Permission denied'));
-
- const params: CodeParserToolParams = { path: dirPath };
- const result = await tool.execute(params, abortSignal);
-
- expect(result.llmContent).toMatch(
- /Error listing or processing directory/,
- );
- expect(result.returnDisplay).toMatch(
- /Error: Failed to process directory./,
- );
- });
- });
-
- describe('requiresConfirmation', () => {
- it('should return null', async () => {
- const params: CodeParserToolParams = { path: 'anypath' };
- expect(await tool.requiresConfirmation(params)).toBeNull();
- });
- });
-});
diff --git a/packages/core/src/tools/code_parser.ts b/packages/core/src/tools/code_parser.ts
deleted file mode 100644
index 12926f29..00000000
--- a/packages/core/src/tools/code_parser.ts
+++ /dev/null
@@ -1,386 +0,0 @@
-/**
- * @license
- * Copyright 2025 Google LLC
- * SPDX-License-Identifier: Apache-2.0
- */
-
-import Parser from 'tree-sitter';
-import Python from 'tree-sitter-python';
-import Java from 'tree-sitter-java';
-import Go from 'tree-sitter-go';
-import CSharp from 'tree-sitter-c-sharp';
-import TreeSitterTypeScript from 'tree-sitter-typescript';
-import Rust from 'tree-sitter-rust'; // Added
-import fs from 'fs/promises';
-import path from 'path';
-import { BaseTool, ToolResult, ToolCallConfirmationDetails } from './tools.js';
-import { SchemaValidator } from '../utils/schemaValidator.js';
-import { makeRelative, shortenPath } from '../utils/paths.js'; // Removed isWithinRoot
-import { Config } from '../config/config.js';
-
-type TreeSitterLanguage = Parameters<typeof Parser.prototype.setLanguage>[0];
-
-export interface CodeParserToolParams {
- path: string;
- ignore?: string[];
- languages?: string[];
-}
-
-export class CodeParserTool extends BaseTool<CodeParserToolParams, ToolResult> {
- static readonly Name = 'code_parser';
-
- private parser: Parser;
-
- constructor(
- private rootDirectory: string,
- private config: Config,
- ) {
- super(
- CodeParserTool.Name,
- 'CodeParser',
- 'Parses the code in the specified directory path or a single file to generate AST representations. This should be used to get a better understanding of the codebase when refactoring and building out new features.',
- {
- properties: {
- path: {
- type: 'string',
- description:
- 'The absolute path to the directory or file to parse (must be absolute, not relative)',
- },
- languages: {
- type: 'array',
- description:
- 'Optional: specific languages to parse (e.g., ["python", "java", "go", "csharp", "typescript", "tsx", "javascript", "rust"]). Defaults to supported languages.',
- items: {
- type: 'string',
- },
- },
- },
- required: ['path'],
- type: 'object',
- },
- );
- this.rootDirectory = path.resolve(rootDirectory);
- this.parser = new Parser();
- }
-
- // Added private isWithinRoot method
- private isWithinRoot(dirpath: string): boolean {
- const normalizedPath = path.normalize(dirpath);
- const normalizedRoot = path.normalize(this.rootDirectory);
- const rootWithSep = normalizedRoot.endsWith(path.sep)
- ? normalizedRoot
- : normalizedRoot + path.sep;
- return (
- normalizedPath === normalizedRoot ||
- normalizedPath.startsWith(rootWithSep)
- );
- }
-
- private getLanguageParser(language: string): TreeSitterLanguage | undefined {
- switch (language.toLowerCase()) {
- case 'python':
- return Python;
- case 'java':
- return Java;
- case 'go':
- return Go;
- case 'csharp':
- return CSharp;
- case 'typescript':
- return TreeSitterTypeScript.typescript;
- case 'tsx':
- return TreeSitterTypeScript.tsx;
- case 'javascript': // Use TypeScript parser for JS as it handles modern JS well
- return TreeSitterTypeScript.typescript;
- case 'rust': // Added
- return Rust; // Added
- default:
- console.warn(
- `Language '${language}' is not supported by the CodeParserTool.`,
- );
- return undefined;
- }
- }
-
- validateToolParams(params: CodeParserToolParams): string | null {
- if (
- this.schema.parameters &&
- !SchemaValidator.validate(
- this.schema.parameters as Record<string, unknown>,
- params,
- )
- ) {
- return 'Parameters failed schema validation.';
- }
- if (!path.isAbsolute(params.path)) {
- return `Path must be absolute: ${params.path}`;
- }
- if (!this.isWithinRoot(params.path)) {
- // Use the class method
- return `Path must be within the root directory (${this.rootDirectory}): ${params.path}`;
- }
- if (
- params.languages &&
- (!Array.isArray(params.languages) ||
- !params.languages.every((lang) => typeof lang === 'string'))
- ) {
- return 'Languages parameter must be an array of strings.';
- }
- return null;
- }
-
- getDescription(params: CodeParserToolParams): string {
- const relativePath = makeRelative(params.path, this.rootDirectory);
- return `Parse ${shortenPath(relativePath)}`;
- }
-
- private errorResult(llmContent: string, returnDisplay: string): ToolResult {
- return {
- llmContent,
- returnDisplay: `Error: ${returnDisplay}`,
- };
- }
-
- private async parseFile(
- filePath: string,
- language: string,
- maxFileSize?: number,
- ): Promise<string | null> {
- const langParser = this.getLanguageParser(language);
- if (!langParser) {
- return null;
- }
- this.parser.setLanguage(langParser);
-
- try {
- const stats = await fs.stat(filePath);
- if (maxFileSize && stats.size > maxFileSize) {
- console.warn(
- `File ${filePath} exceeds maxFileSize (${stats.size} > ${maxFileSize}), skipping.`,
- );
- return null;
- }
-
- const fileContent = await fs.readFile(filePath, 'utf8');
- const tree = this.parser.parse(fileContent);
- return this.formatTree(tree.rootNode, 0);
- } catch (error) {
- console.error(
- `Error parsing file ${filePath} with language ${language}:`,
- error,
- );
- return null;
- }
- }
-
- // Helper function to format the AST similar to the Go version
- private formatTree(node: Parser.SyntaxNode, level: number): string {
- let formattedTree = '';
- const indent = ' '.repeat(level);
- const sexp = node.toString(); // tree-sitter's Node.toString() returns S-expression
- const maxLength = 100;
-
- if (sexp.length < maxLength) {
- // MODIFIED LINE: Removed !sexp.includes('\n')
- formattedTree += `${indent}${sexp}\n`;
- return formattedTree;
- }
-
- // Expand full format if the S-expression is complex or long
- formattedTree += `${indent}(${node.type}\n`;
-
- for (const child of node.namedChildren) {
- formattedTree += this.formatTree(child, level + 1);
- }
-
- // Iterating all children (named and unnamed) to be closer to Go's formatTree.
- // The original Go code iterates `node.NamedChildCount()` and then `node.ChildCount()`
- // which implies it processes named children and then all children (including named again).
- // Here, we iterate named, then iterate all, but skip if already processed as named.
- // This logic might need further refinement if the exact Go output for unnamed nodes is critical.
- // For now, focusing on named children as per the Go code's primary loop in formatTree.
- // If a more exact match for unnamed nodes is needed, the iteration logic for `node.children`
- // and skipping already processed namedChildren would be added here.
-
- formattedTree += `${indent})\n`;
- return formattedTree;
- }
-
- private getFileLanguage(filePath: string): string | undefined {
- const extension = path.extname(filePath).toLowerCase();
- switch (extension) {
- case '.py':
- return 'python';
- case '.java':
- return 'java';
- case '.go':
- return 'go';
- case '.cs':
- return 'csharp';
- case '.ts':
- return 'typescript';
- case '.tsx':
- return 'tsx';
- case '.js':
- return 'javascript';
- case '.jsx': // Treat jsx as tsx for parsing
- return 'tsx';
- case '.mjs':
- return 'javascript';
- case '.cjs':
- return 'javascript';
- case '.rs': // Added
- return 'rust'; // Added
- default:
- return undefined;
- }
- }
-
- async execute(
- params: CodeParserToolParams,
- _signal: AbortSignal,
- ): Promise<ToolResult> {
- const validationError = this.validateToolParams(params);
- if (validationError) {
- return this.errorResult(
- `Error: Invalid parameters provided. Reason: ${validationError}`,
- 'Failed to execute tool.',
- );
- }
-
- const targetPath = params.path;
- let stats;
- try {
- stats = await fs.stat(targetPath);
- } catch (error) {
- if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
- return this.errorResult(
- `Error: Path not found or inaccessible: ${targetPath}`,
- 'Path not found or inaccessible.',
- );
- }
- return this.errorResult(
- `Error: Cannot access path: ${(error as Error).message}`,
- 'Cannot access path.',
- );
- }
-
- const defaultLanguages = [
- 'python',
- 'java',
- 'go',
- 'csharp',
- 'typescript',
- 'tsx',
- 'javascript',
- 'rust', // Added
- ];
- const languagesToParse = (
- params.languages && params.languages.length > 0
- ? params.languages
- : defaultLanguages
- ).map((lang) => lang.toLowerCase());
- const maxFileSize = 1024 * 1024; // 1MB
-
- const supportedLanguagesToParse = languagesToParse.filter((lang) =>
- this.getLanguageParser(lang),
- );
- if (supportedLanguagesToParse.length === 0) {
- const availableLangs =
- defaultLanguages
- .filter((lang) => this.getLanguageParser(lang))
- .join(', ') || 'none configured';
- return this.errorResult(
- `Error: No supported languages specified for parsing. Requested: ${languagesToParse.join(', ') || 'default'}. Available: ${availableLangs}.`,
- 'No supported languages to parse.',
- );
- }
-
- let parsedCodeOutput = '';
- let filesProcessedCount = 0;
-
- if (stats.isDirectory()) {
- try {
- const files = await fs.readdir(targetPath);
- if (files.length === 0) {
- return {
- llmContent: `Directory ${targetPath} is empty.`,
- returnDisplay: 'Directory is empty.',
- };
- }
-
- for (const file of files) {
- const filePath = path.join(targetPath, file);
- let fileStats;
- try {
- fileStats = await fs.stat(filePath);
- } catch {
- console.warn(`Could not stat file ${filePath}, skipping.`);
- continue;
- }
-
- if (fileStats.isFile()) {
- const fileLang = this.getFileLanguage(filePath);
- if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
- const ast = await this.parseFile(filePath, fileLang, maxFileSize);
- if (ast) {
- parsedCodeOutput += `-------------${filePath}-------------\n`;
- parsedCodeOutput += ast + '\n';
- filesProcessedCount++;
- }
- }
- }
- }
- } catch (error) {
- return this.errorResult(
- `Error listing or processing directory ${targetPath}: ${(error as Error).message}`,
- 'Failed to process directory.',
- );
- }
- } else if (stats.isFile()) {
- const fileLang = this.getFileLanguage(targetPath);
- if (fileLang && supportedLanguagesToParse.includes(fileLang)) {
- const ast = await this.parseFile(targetPath, fileLang, maxFileSize);
- if (ast) {
- parsedCodeOutput += `-------------${targetPath}-------------\n`;
- parsedCodeOutput += ast + '\n';
- filesProcessedCount++;
- } else {
- return this.errorResult(
- `Error: Could not parse file ${targetPath}. Language '${fileLang}' is supported but parsing failed. Check logs.`,
- 'Failed to parse file.',
- );
- }
- } else {
- return this.errorResult(
- `Error: File ${targetPath} is not of a supported language type for parsing or language not specified. Supported: ${supportedLanguagesToParse.join(', ')}. Detected extension for language: ${fileLang || 'unknown'}.`,
- 'Unsupported file type or language.',
- );
- }
- } else {
- return this.errorResult(
- `Error: Path is not a file or directory: ${targetPath}`,
- 'Path is not a file or directory.',
- );
- }
-
- if (filesProcessedCount === 0) {
- return {
- llmContent: `No files were parsed in ${targetPath}. Ensure files match supported languages (${supportedLanguagesToParse.join(', ')}), are not empty or too large, and are not ignored.`,
- returnDisplay: 'No files parsed.',
- };
- }
-
- const returnDisplay = `Parsed ${filesProcessedCount} file(s).`;
- return {
- llmContent: `Parsed code from ${targetPath}:\n${parsedCodeOutput}`,
- returnDisplay,
- };
- }
-
- async requiresConfirmation(
- _params: CodeParserToolParams,
- ): Promise<ToolCallConfirmationDetails | null> {
- return null;
- }
-}