From 0ef9c0b792f056015d0eee4486480c1f80015fc5 Mon Sep 17 00:00:00 2001 From: Sandy Tao Date: Wed, 23 Jul 2025 22:37:28 -0700 Subject: Log prompt id when a loop is detected (#4765) Co-authored-by: N. Taylor Mullen --- packages/core/src/core/client.ts | 2 +- packages/core/src/services/loopDetectionService.test.ts | 10 +++++----- packages/core/src/services/loopDetectionService.ts | 16 ++++++++++++---- .../src/telemetry/clearcut-logger/clearcut-logger.ts | 4 ++-- packages/core/src/telemetry/types.ts | 4 +++- 5 files changed, 23 insertions(+), 13 deletions(-) (limited to 'packages/core/src') diff --git a/packages/core/src/core/client.ts b/packages/core/src/core/client.ts index 49b63c8b..6f482307 100644 --- a/packages/core/src/core/client.ts +++ b/packages/core/src/core/client.ts @@ -293,7 +293,7 @@ export class GeminiClient { originalModel?: string, ): AsyncGenerator { if (this.lastPromptId !== prompt_id) { - this.loopDetector.reset(); + this.loopDetector.reset(prompt_id); this.lastPromptId = prompt_id; } this.sessionTurnCount++; diff --git a/packages/core/src/services/loopDetectionService.test.ts b/packages/core/src/services/loopDetectionService.test.ts index bf3afd32..b2863168 100644 --- a/packages/core/src/services/loopDetectionService.test.ts +++ b/packages/core/src/services/loopDetectionService.test.ts @@ -168,21 +168,21 @@ describe('LoopDetectionService', () => { ); } - service.reset(); + service.reset(''); for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { expect(service.addAndCheck(createContentEvent('obj.method()'))).toBe( false, ); } - service.reset(); + service.reset(''); for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { expect( service.addAndCheck(createContentEvent('arr.filter().map()')), ).toBe(false); } - service.reset(); + service.reset(''); for (let i = 0; i < CONTENT_LOOP_THRESHOLD + 2; i++) { expect( service.addAndCheck( @@ -203,7 +203,7 @@ describe('LoopDetectionService', () => { service.addAndCheck(createContentEvent('This is a sentence.')), ).toBe(true); - service.reset(); + service.reset(''); for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) { expect( service.addAndCheck(createContentEvent('Is this a question? ')), @@ -213,7 +213,7 @@ describe('LoopDetectionService', () => { service.addAndCheck(createContentEvent('Is this a question? ')), ).toBe(true); - service.reset(); + service.reset(''); for (let i = 0; i < CONTENT_LOOP_THRESHOLD - 1; i++) { expect( service.addAndCheck(createContentEvent('What excitement!\n')), diff --git a/packages/core/src/services/loopDetectionService.ts b/packages/core/src/services/loopDetectionService.ts index 1c3ea412..85f38c12 100644 --- a/packages/core/src/services/loopDetectionService.ts +++ b/packages/core/src/services/loopDetectionService.ts @@ -50,6 +50,7 @@ const SENTENCE_ENDING_PUNCTUATION_REGEX = /[.!?]+(?=\s|$)/; */ export class LoopDetectionService { private readonly config: Config; + private promptId = ''; // Tool call tracking private lastToolCallKey: string | null = null; @@ -129,7 +130,10 @@ export class LoopDetectionService { if (this.toolCallRepetitionCount >= TOOL_CALL_LOOP_THRESHOLD) { logLoopDetected( this.config, - new LoopDetectedEvent(LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS), + new LoopDetectedEvent( + LoopType.CONSECUTIVE_IDENTICAL_TOOL_CALLS, + this.promptId, + ), ); return true; } @@ -170,7 +174,10 @@ export class LoopDetectionService { if (this.sentenceRepetitionCount >= CONTENT_LOOP_THRESHOLD) { logLoopDetected( this.config, - new LoopDetectedEvent(LoopType.CHANTING_IDENTICAL_SENTENCES), + new LoopDetectedEvent( + LoopType.CHANTING_IDENTICAL_SENTENCES, + this.promptId, + ), ); return true; } @@ -234,7 +241,7 @@ Please analyze the conversation history to determine the possibility that the co } logLoopDetected( this.config, - new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP), + new LoopDetectedEvent(LoopType.LLM_DETECTED_LOOP, this.promptId), ); return true; } else { @@ -251,7 +258,8 @@ Please analyze the conversation history to determine the possibility that the co /** * Resets all loop detection state. */ - reset(): void { + reset(promptId: string): void { + this.promptId = promptId; this.resetToolCallCount(); this.resetSentenceCount(); this.resetLlmCheckTracking(); diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts index ba47e7a0..d36a16b5 100644 --- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts +++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts @@ -481,8 +481,8 @@ export class ClearcutLogger { logLoopDetectedEvent(event: LoopDetectedEvent): void { const data = [ { - gemini_cli_key: EventMetadataKey.GEMINI_CLI_SESSION_ID, - value: this.config?.getSessionId() ?? '', + gemini_cli_key: EventMetadataKey.GEMINI_CLI_PROMPT_ID, + value: JSON.stringify(event.prompt_id), }, { gemini_cli_key: EventMetadataKey.GEMINI_CLI_LOOP_DETECTED_TYPE, diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts index 268457b5..69dffb08 100644 --- a/packages/core/src/telemetry/types.ts +++ b/packages/core/src/telemetry/types.ts @@ -256,11 +256,13 @@ export class LoopDetectedEvent { 'event.name': 'loop_detected'; 'event.timestamp': string; // ISO 8601 loop_type: LoopType; + prompt_id: string; - constructor(loop_type: LoopType) { + constructor(loop_type: LoopType, prompt_id: string) { this['event.name'] = 'loop_detected'; this['event.timestamp'] = new Date().toISOString(); this.loop_type = loop_type; + this.prompt_id = prompt_id; } } -- cgit v1.2.3