Adds Flash Fallback logging and clearcut logging (#3843)

8 files changed, 93 insertions, 1 deletions

diff --git a/docs/telemetry.md b/docs/telemetry.md
index b308217f..76958794 100644
--- a/docs/telemetry.md
+++ b/docs/telemetry.md
@@ -194,6 +194,10 @@ Logs are timestamped records of specific events. The following events are logged
     - `response_text` (if applicable)
     - `auth_type`
 
+- `gemini_cli.flash_fallback`: This event occurs when Gemini CLI switches to flash as fallback.
+  - **Attributes**:
+    - `auth_type`
+
 ### Metrics
 
 Metrics are numerical measurements of behavior over time. The following metrics are collected for Gemini CLI:
diff --git a/packages/cli/src/ui/App.tsx b/packages/cli/src/ui/App.tsx
index 519a4b53..f2324b28 100644
--- a/packages/cli/src/ui/App.tsx
+++ b/packages/cli/src/ui/App.tsx
@@ -54,6 +54,8 @@ import {
   ApprovalMode,
   isEditorAvailable,
   EditorType,
+  FlashFallbackEvent,
+  logFlashFallback,
 } from '@google/gemini-cli-core';
 import { validateAuthMethod } from '../config/auth.js';
 import { useLogger } from './hooks/useLogger.js';
@@ -340,6 +342,10 @@ const App = ({ config, settings, startupWarnings = [], version }: AppProps) => {
       config.setQuotaErrorOccurred(true);
       // Switch model for future use but return false to stop current retry
       config.setModel(fallbackModel);
+      logFlashFallback(
+        config,
+        new FlashFallbackEvent(config.getContentGeneratorConfig().authType!),
+      );
       return false; // Don't continue with current prompt
     };
 
diff --git a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
index 42b1f6fc..cd2abe81 100644
--- a/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
+++ b/packages/core/src/telemetry/clearcut-logger/clearcut-logger.ts
@@ -14,6 +14,7 @@ import {
   ApiRequestEvent,
   ApiResponseEvent,
   ApiErrorEvent,
+  FlashFallbackEvent,
 } from '../types.js';
 import { EventMetadataKey } from './event-metadata-key.js';
 import { Config } from '../../config/config.js';
@@ -30,6 +31,7 @@ const api_request_event_name = 'api_request';
 const api_response_event_name = 'api_response';
 const api_error_event_name = 'api_error';
 const end_session_event_name = 'end_session';
+const flash_fallback_event_name = 'flash_fallback';
 
 export interface LogResponse {
   nextRequestWaitMs?: number;
@@ -431,6 +433,20 @@ export class ClearcutLogger {
     this.flushIfNeeded();
   }
 
+  logFlashFallbackEvent(event: FlashFallbackEvent): void {
+    const data = [
+      {
+        gemini_cli_key: EventMetadataKey.GEMINI_CLI_AUTH_TYPE,
+        value: JSON.stringify(event.auth_type),
+      },
+    ];
+
+    this.enqueueLogEvent(this.createLogEvent(flash_fallback_event_name, data));
+    this.flushToClearcut().catch((error) => {
+      console.debug('Error flushing to Clearcut:', error);
+    });
+  }
+
   logEndSessionEvent(event: EndSessionEvent): void {
     const data = [
       {
diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts
index de760205..62c4bf24 100644
--- a/packages/core/src/telemetry/constants.ts
+++ b/packages/core/src/telemetry/constants.ts
@@ -12,6 +12,7 @@ export const EVENT_API_REQUEST = 'gemini_cli.api_request';
 export const EVENT_API_ERROR = 'gemini_cli.api_error';
 export const EVENT_API_RESPONSE = 'gemini_cli.api_response';
 export const EVENT_CLI_CONFIG = 'gemini_cli.config';
+export const EVENT_FLASH_FALLBACK = 'gemini_cli.flash_fallback';
 
 export const METRIC_TOOL_CALL_COUNT = 'gemini_cli.tool.call.count';
 export const METRIC_TOOL_CALL_LATENCY = 'gemini_cli.tool.call.latency';
diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts
index a17c8af3..8da31727 100644
--- a/packages/core/src/telemetry/index.ts
+++ b/packages/core/src/telemetry/index.ts
@@ -25,6 +25,7 @@ export {
   logApiRequest,
   logApiError,
   logApiResponse,
+  logFlashFallback,
 } from './loggers.js';
 export {
   StartSessionEvent,
@@ -35,6 +36,7 @@ export {
   ApiErrorEvent,
   ApiResponseEvent,
   TelemetryEvent,
+  FlashFallbackEvent,
 } from './types.js';
 export { SpanStatusCode, ValueType } from '@opentelemetry/api';
 export { SemanticAttributes } from '@opentelemetry/semantic-conventions';
diff --git a/packages/core/src/telemetry/loggers.test.ts b/packages/core/src/telemetry/loggers.test.ts
index 13617946..7a24bcca 100644
--- a/packages/core/src/telemetry/loggers.test.ts
+++ b/packages/core/src/telemetry/loggers.test.ts
@@ -23,6 +23,7 @@ import {
   EVENT_CLI_CONFIG,
   EVENT_TOOL_CALL,
   EVENT_USER_PROMPT,
+  EVENT_FLASH_FALLBACK,
 } from './constants.js';
 import {
   logApiRequest,
@@ -30,6 +31,7 @@ import {
   logCliConfiguration,
   logUserPrompt,
   logToolCall,
+  logFlashFallback,
 } from './loggers.js';
 import {
   ApiRequestEvent,
@@ -38,6 +40,7 @@ import {
   ToolCallDecision,
   ToolCallEvent,
   UserPromptEvent,
+  FlashFallbackEvent,
 } from './types.js';
 import * as metrics from './metrics.js';
 import * as sdk from './sdk.js';
@@ -350,6 +353,29 @@ describe('loggers', () => {
     });
   });
 
+  describe('logFlashFallback', () => {
+    const mockConfig = {
+      getSessionId: () => 'test-session-id',
+      getUsageStatisticsEnabled: () => true,
+    } as unknown as Config;
+
+    it('should log flash fallback event', () => {
+      const event = new FlashFallbackEvent(AuthType.USE_VERTEX_AI);
+
+      logFlashFallback(mockConfig, event);
+
+      expect(mockLogger.emit).toHaveBeenCalledWith({
+        body: 'Switching to flash as Fallback.',
+        attributes: {
+          'session.id': 'test-session-id',
+          'event.name': EVENT_FLASH_FALLBACK,
+          'event.timestamp': '2025-01-01T00:00:00.000Z',
+          auth_type: 'vertex-ai',
+        },
+      });
+    });
+  });
+
   describe('logToolCall', () => {
     const cfg1 = {
       getSessionId: () => 'test-session-id',
diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts
index a7231e2f..5929ec58 100644
--- a/packages/core/src/telemetry/loggers.ts
+++ b/packages/core/src/telemetry/loggers.ts
@@ -14,6 +14,7 @@ import {
   EVENT_CLI_CONFIG,
   EVENT_TOOL_CALL,
   EVENT_USER_PROMPT,
+  EVENT_FLASH_FALLBACK,
   SERVICE_NAME,
 } from './constants.js';
 import {
@@ -23,6 +24,7 @@ import {
   StartSessionEvent,
   ToolCallEvent,
   UserPromptEvent,
+  FlashFallbackEvent,
 } from './types.js';
 import {
   recordApiErrorMetrics,
@@ -156,6 +158,28 @@ export function logApiRequest(config: Config, event: ApiRequestEvent): void {
   logger.emit(logRecord);
 }
 
+export function logFlashFallback(
+  config: Config,
+  event: FlashFallbackEvent,
+): void {
+  ClearcutLogger.getInstance(config)?.logFlashFallbackEvent(event);
+  if (!isTelemetrySdkInitialized()) return;
+
+  const attributes: LogAttributes = {
+    ...getCommonAttributes(config),
+    ...event,
+    'event.name': EVENT_FLASH_FALLBACK,
+    'event.timestamp': new Date().toISOString(),
+  };
+
+  const logger = logs.getLogger(SERVICE_NAME);
+  const logRecord: LogRecord = {
+    body: `Switching to flash as Fallback.`,
+    attributes,
+  };
+  logger.emit(logRecord);
+}
+
 export function logApiError(config: Config, event: ApiErrorEvent): void {
   const uiEvent = {
     ...event,
diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts
index 9ac12050..d04756e2 100644
--- a/packages/core/src/telemetry/types.ts
+++ b/packages/core/src/telemetry/types.ts
@@ -234,6 +234,18 @@ export class ApiResponseEvent {
   }
 }
 
+export class FlashFallbackEvent {
+  'event.name': 'flash_fallback';
+  'event.timestamp': string; // ISO 8601
+  auth_type: string;
+
+  constructor(auth_type: string) {
+    this['event.name'] = 'flash_fallback';
+    this['event.timestamp'] = new Date().toISOString();
+    this.auth_type = auth_type;
+  }
+}
+
 export type TelemetryEvent =
   | StartSessionEvent
   | EndSessionEvent
@@ -241,4 +253,5 @@ export type TelemetryEvent =
   | ToolCallEvent
   | ApiRequestEvent
   | ApiErrorEvent
-  | ApiResponseEvent;
+  | ApiResponseEvent
+  | FlashFallbackEvent;