diff options
| author | Jerop Kipruto <[email protected]> | 2025-06-12 16:48:10 -0400 |
|---|---|---|
| committer | GitHub <[email protected]> | 2025-06-12 20:48:10 +0000 |
| commit | 6723c72fa5468be713c05205c75be532729e8f92 (patch) | |
| tree | 2392b344fb942f1c452e9fca5b5b6d131a827805 /packages/core/src/telemetry/loggers.test.ts | |
| parent | f8863f4d00f23a3e29496535be6cf0bb80ee43e9 (diff) | |
telemetry: include user decisions in tool call logs (#966)
Add the user's decision (accept, reject, modify) to tool call telemetry to better understand user intent. The decision provides crucial context to the `success` metric, as a user can reject a call that would have succeeded or accept one that fails.
Also prettify the arguments json.
Example:

#750
Diffstat (limited to 'packages/core/src/telemetry/loggers.test.ts')
| -rw-r--r-- | packages/core/src/telemetry/loggers.test.ts | 238 |
1 files changed, 238 insertions, 0 deletions
diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts index 2153ef48..a09f3eaf 100644 --- a/packages/core/src/telemetry/loggers.test.ts +++ b/packages/core/src/telemetry/loggers.test.ts @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ +import { ToolConfirmationOutcome } from '../index.js'; import { logs } from '@opentelemetry/api-logs'; import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; import { Config } from '../config/config.js'; @@ -12,6 +13,8 @@ import { logApiResponse, logCliConfiguration, logUserPrompt, + logToolCall, + ToolCallDecision, } from './loggers.js'; import * as metrics from './metrics.js'; import * as sdk from './sdk.js'; @@ -236,4 +239,239 @@ describe('loggers', () => { }); }); }); + + describe('logToolCall', () => { + const mockConfig = { + getSessionId: () => 'test-session-id', + } as Config; + + const mockMetrics = { + recordToolCallMetrics: vi.fn(), + }; + + beforeEach(() => { + vi.spyOn(metrics, 'recordToolCallMetrics').mockImplementation( + mockMetrics.recordToolCallMetrics, + ); + mockLogger.emit.mockReset(); + }); + + it('should log a tool call with all fields', () => { + const event = { + function_name: 'test-function', + function_args: { + arg1: 'value1', + arg2: 2, + }, + duration_ms: 100, + success: true, + }; + + logToolCall(mockConfig, event, ToolConfirmationOutcome.ProceedOnce); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: test-function. Decision: accept. Success: true. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.tool_call', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + function_name: 'test-function', + function_args: JSON.stringify( + { + arg1: 'value1', + arg2: 2, + }, + null, + 2, + ), + duration_ms: 100, + success: true, + decision: ToolCallDecision.ACCEPT, + }, + }); + + expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-function', + 100, + true, + ToolCallDecision.ACCEPT, + ); + }); + it('should log a tool call with a reject decision', () => { + const event = { + function_name: 'test-function', + function_args: { + arg1: 'value1', + arg2: 2, + }, + duration_ms: 100, + success: false, + }; + + logToolCall(mockConfig, event, ToolConfirmationOutcome.Cancel); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: test-function. Decision: reject. Success: false. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.tool_call', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + function_name: 'test-function', + function_args: JSON.stringify( + { + arg1: 'value1', + arg2: 2, + }, + null, + 2, + ), + duration_ms: 100, + success: false, + decision: ToolCallDecision.REJECT, + }, + }); + + expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-function', + 100, + false, + ToolCallDecision.REJECT, + ); + }); + + it('should log a tool call with a modify decision', () => { + const event = { + function_name: 'test-function', + function_args: { + arg1: 'value1', + arg2: 2, + }, + duration_ms: 100, + success: true, + }; + + logToolCall(mockConfig, event, ToolConfirmationOutcome.ModifyWithEditor); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: test-function. Decision: modify. Success: true. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.tool_call', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + function_name: 'test-function', + function_args: JSON.stringify( + { + arg1: 'value1', + arg2: 2, + }, + null, + 2, + ), + duration_ms: 100, + success: true, + decision: ToolCallDecision.MODIFY, + }, + }); + + expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-function', + 100, + true, + ToolCallDecision.MODIFY, + ); + }); + + it('should log a tool call without a decision', () => { + const event = { + function_name: 'test-function', + function_args: { + arg1: 'value1', + arg2: 2, + }, + duration_ms: 100, + success: true, + }; + + logToolCall(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: test-function. Success: true. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.tool_call', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + function_name: 'test-function', + function_args: JSON.stringify( + { + arg1: 'value1', + arg2: 2, + }, + null, + 2, + ), + duration_ms: 100, + success: true, + }, + }); + + expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-function', + 100, + true, + undefined, + ); + }); + + it('should log a failed tool call with an error', () => { + const event = { + function_name: 'test-function', + function_args: { + arg1: 'value1', + arg2: 2, + }, + duration_ms: 100, + success: false, + error: 'test-error', + error_type: 'test-error-type', + }; + + logToolCall(mockConfig, event); + + expect(mockLogger.emit).toHaveBeenCalledWith({ + body: 'Tool call: test-function. Success: false. Duration: 100ms.', + attributes: { + 'session.id': 'test-session-id', + 'event.name': 'gemini_cli.tool_call', + 'event.timestamp': '2025-01-01T00:00:00.000Z', + function_name: 'test-function', + function_args: JSON.stringify( + { + arg1: 'value1', + arg2: 2, + }, + null, + 2, + ), + duration_ms: 100, + success: false, + error: 'test-error', + 'error.message': 'test-error', + error_type: 'test-error-type', + 'error.type': 'test-error-type', + }, + }); + + expect(mockMetrics.recordToolCallMetrics).toHaveBeenCalledWith( + mockConfig, + 'test-function', + 100, + false, + undefined, + ); + }); + }); }); |
