diff options
Diffstat (limited to 'packages/core/src/telemetry')
| -rw-r--r-- | packages/core/src/telemetry/constants.ts | 24 | ||||
| -rw-r--r-- | packages/core/src/telemetry/index.ts | 31 | ||||
| -rw-r--r-- | packages/core/src/telemetry/loggers.ts | 191 | ||||
| -rw-r--r-- | packages/core/src/telemetry/metrics.ts | 145 | ||||
| -rw-r--r-- | packages/core/src/telemetry/sdk.ts | 128 | ||||
| -rw-r--r-- | packages/core/src/telemetry/types.ts | 73 |
6 files changed, 592 insertions, 0 deletions
diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts new file mode 100644 index 00000000..67d5b38b --- /dev/null +++ b/packages/core/src/telemetry/constants.ts @@ -0,0 +1,24 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { randomUUID } from 'crypto'; + +export const SERVICE_NAME = 'gemini-code'; +export const sessionId = randomUUID(); + +export const EVENT_USER_PROMPT = 'gemini_code.user_prompt'; +export const EVENT_TOOL_CALL = 'gemini_code.tool_call'; +export const EVENT_API_REQUEST = 'gemini_code.api_request'; +export const EVENT_API_ERROR = 'gemini_code.api_error'; +export const EVENT_API_RESPONSE = 'gemini_code.api_response'; +export const EVENT_CLI_CONFIG = 'gemini_code.config'; + +export const METRIC_TOOL_CALL_COUNT = 'gemini_code.tool.call.count'; +export const METRIC_TOOL_CALL_LATENCY = 'gemini_code.tool.call.latency'; +export const METRIC_API_REQUEST_COUNT = 'gemini_code.api.request.count'; +export const METRIC_API_REQUEST_LATENCY = 'gemini_code.api.request.latency'; +export const METRIC_TOKEN_INPUT_COUNT = 'gemini_code.token.input.count'; +export const METRIC_SESSION_COUNT = 'gemini_code.session.count'; diff --git a/packages/core/src/telemetry/index.ts b/packages/core/src/telemetry/index.ts new file mode 100644 index 00000000..7b2ab0e7 --- /dev/null +++ b/packages/core/src/telemetry/index.ts @@ -0,0 +1,31 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +export { + initializeTelemetry, + shutdownTelemetry, + isTelemetrySdkInitialized, +} from './sdk.js'; +export { + logCliConfiguration, + logUserPrompt, + logToolCall, + logApiRequest, + logApiError, + logApiResponse, +} from './loggers.js'; +export { + UserPromptEvent, + ToolCallEvent, + ApiRequestEvent, + ApiErrorEvent, + ApiResponseEvent, + CliConfigEvent, + TelemetryEvent, +} from './types.js'; +export { SpanStatusCode, ValueType } from '@opentelemetry/api'; +export { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +export { sessionId } from './constants.js'; diff --git a/packages/core/src/telemetry/loggers.ts b/packages/core/src/telemetry/loggers.ts new file mode 100644 index 00000000..ccab12ff --- /dev/null +++ b/packages/core/src/telemetry/loggers.ts @@ -0,0 +1,191 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { logs, LogRecord, LogAttributes } from '@opentelemetry/api-logs'; +import { SemanticAttributes } from '@opentelemetry/semantic-conventions'; +import { Config } from '../config/config.js'; +import { + EVENT_API_ERROR, + EVENT_API_REQUEST, + EVENT_API_RESPONSE, + EVENT_CLI_CONFIG, + EVENT_TOOL_CALL, + EVENT_USER_PROMPT, + SERVICE_NAME, +} from './constants.js'; +import { + ApiErrorEvent, + ApiRequestEvent, + ApiResponseEvent, + ToolCallEvent, + UserPromptEvent, +} from './types.js'; +import { + recordApiErrorMetrics, + recordApiRequestMetrics, + recordApiResponseMetrics, + recordToolCallMetrics, +} from './metrics.js'; +import { isTelemetrySdkInitialized } from './sdk.js'; + +const shouldLogUserPrompts = (config: Config): boolean => + config.getTelemetryLogUserPromptsEnabled() ?? false; + +export function logCliConfiguration(config: Config): void { + if (!isTelemetrySdkInitialized()) return; + + const attributes: LogAttributes = { + 'event.name': EVENT_CLI_CONFIG, + 'event.timestamp': new Date().toISOString(), + model: config.getModel(), + sandbox_enabled: + typeof config.getSandbox() === 'string' ? true : config.getSandbox(), + core_tools_enabled: (config.getCoreTools() ?? []).join(','), + approval_mode: config.getApprovalMode(), + vertex_ai_enabled: config.getVertexAI() ?? false, + log_user_prompts_enabled: config.getTelemetryLogUserPromptsEnabled(), + file_filtering_respect_git_ignore: + config.getFileFilteringRespectGitIgnore(), + file_filtering_allow_build_artifacts: + config.getFileFilteringAllowBuildArtifacts(), + }; + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: 'CLI configuration loaded.', + attributes, + }; + logger.emit(logRecord); +} + +export function logUserPrompt( + config: Config, + event: Omit<UserPromptEvent, 'event.name' | 'event.timestamp' | 'prompt'> & { + prompt: string; + }, +): void { + if (!isTelemetrySdkInitialized()) return; + const { prompt, ...restOfEventArgs } = event; + const attributes: LogAttributes = { + ...restOfEventArgs, + 'event.name': EVENT_USER_PROMPT, + 'event.timestamp': new Date().toISOString(), + }; + if (shouldLogUserPrompts(config)) { + attributes.prompt = prompt; + } + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `User prompt. Length: ${event.prompt_char_count}`, + attributes, + }; + logger.emit(logRecord); +} + +export function logToolCall( + event: Omit<ToolCallEvent, 'event.name' | 'event.timestamp'>, +): void { + if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { + ...event, + 'event.name': EVENT_TOOL_CALL, + 'event.timestamp': new Date().toISOString(), + function_args: JSON.stringify(event.function_args), + }; + if (event.error) { + attributes['error.message'] = event.error; + if (event.error_type) { + attributes['error.type'] = event.error_type; + } + } + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `Tool call: ${event.function_name}. Success: ${event.success}. Duration: ${event.duration_ms}ms.`, + attributes, + }; + logger.emit(logRecord); + recordToolCallMetrics(event.function_name, event.duration_ms, event.success); +} + +export function logApiRequest( + event: Omit<ApiRequestEvent, 'event.name' | 'event.timestamp'>, +): void { + if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { + ...event, + 'event.name': EVENT_API_REQUEST, + 'event.timestamp': new Date().toISOString(), + }; + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `API request to ${event.model}. Tokens: ${event.prompt_token_count}.`, + attributes, + }; + logger.emit(logRecord); + recordApiRequestMetrics(event.model, event.prompt_token_count); +} + +export function logApiError( + event: Omit<ApiErrorEvent, 'event.name' | 'event.timestamp'>, +): void { + if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { + ...event, + 'event.name': EVENT_API_ERROR, + 'event.timestamp': new Date().toISOString(), + ['error.message']: event.error, + }; + + if (event.error_type) { + attributes['error.type'] = event.error_type; + } + if (typeof event.status_code === 'number') { + attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code; + } + + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `API error for ${event.model}. Error: ${event.error}. Duration: ${event.duration_ms}ms.`, + attributes, + }; + logger.emit(logRecord); + recordApiErrorMetrics( + event.model, + event.duration_ms, + event.status_code, + event.error_type, + ); +} + +export function logApiResponse( + event: Omit<ApiResponseEvent, 'event.name' | 'event.timestamp'>, +): void { + if (!isTelemetrySdkInitialized()) return; + const attributes: LogAttributes = { + ...event, + 'event.name': EVENT_API_RESPONSE, + 'event.timestamp': new Date().toISOString(), + }; + if (event.error) { + attributes['error.message'] = event.error; + } else if (event.status_code) { + if (typeof event.status_code === 'number') { + attributes[SemanticAttributes.HTTP_STATUS_CODE] = event.status_code; + } + } + + const logger = logs.getLogger(SERVICE_NAME); + const logRecord: LogRecord = { + body: `API response from ${event.model}. Status: ${event.status_code || 'N/A'}. Duration: ${event.duration_ms}ms.`, + attributes, + }; + logger.emit(logRecord); + recordApiResponseMetrics( + event.model, + event.duration_ms, + event.status_code, + event.error, + ); +} diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts new file mode 100644 index 00000000..2e6bd909 --- /dev/null +++ b/packages/core/src/telemetry/metrics.ts @@ -0,0 +1,145 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + metrics, + Attributes, + ValueType, + Meter, + Counter, + Histogram, +} from '@opentelemetry/api'; +import { + SERVICE_NAME, + METRIC_TOOL_CALL_COUNT, + METRIC_TOOL_CALL_LATENCY, + METRIC_API_REQUEST_COUNT, + METRIC_API_REQUEST_LATENCY, + METRIC_TOKEN_INPUT_COUNT, + METRIC_SESSION_COUNT, +} from './constants.js'; + +let cliMeter: Meter | undefined; +let toolCallCounter: Counter | undefined; +let toolCallLatencyHistogram: Histogram | undefined; +let apiRequestCounter: Counter | undefined; +let apiRequestLatencyHistogram: Histogram | undefined; +let tokenInputCounter: Counter | undefined; +let isMetricsInitialized = false; + +export function getMeter(): Meter | undefined { + if (!cliMeter) { + cliMeter = metrics.getMeter(SERVICE_NAME); + } + return cliMeter; +} + +export function initializeMetrics(): void { + if (isMetricsInitialized) return; + + const meter = getMeter(); + if (!meter) return; + + toolCallCounter = meter.createCounter(METRIC_TOOL_CALL_COUNT, { + description: 'Counts tool calls, tagged by function name and success.', + valueType: ValueType.INT, + }); + toolCallLatencyHistogram = meter.createHistogram(METRIC_TOOL_CALL_LATENCY, { + description: 'Latency of tool calls in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + }); + apiRequestCounter = meter.createCounter(METRIC_API_REQUEST_COUNT, { + description: 'Counts API requests, tagged by model and status.', + valueType: ValueType.INT, + }); + apiRequestLatencyHistogram = meter.createHistogram( + METRIC_API_REQUEST_LATENCY, + { + description: 'Latency of API requests in milliseconds.', + unit: 'ms', + valueType: ValueType.INT, + }, + ); + tokenInputCounter = meter.createCounter(METRIC_TOKEN_INPUT_COUNT, { + description: 'Counts the total number of input tokens sent to the API.', + valueType: ValueType.INT, + }); + + const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, { + description: 'Count of CLI sessions started.', + valueType: ValueType.INT, + }); + sessionCounter.add(1); + isMetricsInitialized = true; +} + +export function recordToolCallMetrics( + functionName: string, + durationMs: number, + success: boolean, +): void { + if (!toolCallCounter || !toolCallLatencyHistogram || !isMetricsInitialized) + return; + + const metricAttributes: Attributes = { + function_name: functionName, + success, + }; + toolCallCounter.add(1, metricAttributes); + toolCallLatencyHistogram.record(durationMs, { + function_name: functionName, + }); +} + +export function recordApiRequestMetrics( + model: string, + inputTokenCount: number, +): void { + if (!tokenInputCounter || !isMetricsInitialized) return; + tokenInputCounter.add(inputTokenCount, { model }); +} + +export function recordApiResponseMetrics( + model: string, + durationMs: number, + statusCode?: number | string, + error?: string, +): void { + if ( + !apiRequestCounter || + !apiRequestLatencyHistogram || + !isMetricsInitialized + ) + return; + const metricAttributes: Attributes = { + model, + status_code: statusCode ?? (error ? 'error' : 'ok'), + }; + apiRequestCounter.add(1, metricAttributes); + apiRequestLatencyHistogram.record(durationMs, { model }); +} + +export function recordApiErrorMetrics( + model: string, + durationMs: number, + statusCode?: number | string, + errorType?: string, +): void { + if ( + !apiRequestCounter || + !apiRequestLatencyHistogram || + !isMetricsInitialized + ) + return; + const metricAttributes: Attributes = { + model, + status_code: statusCode ?? 'error', + error_type: errorType ?? 'unknown', + }; + apiRequestCounter.add(1, metricAttributes); + apiRequestLatencyHistogram.record(durationMs, { model }); +} diff --git a/packages/core/src/telemetry/sdk.ts b/packages/core/src/telemetry/sdk.ts new file mode 100644 index 00000000..b55cb149 --- /dev/null +++ b/packages/core/src/telemetry/sdk.ts @@ -0,0 +1,128 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { DiagConsoleLogger, DiagLogLevel, diag } from '@opentelemetry/api'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-grpc'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc'; +import { NodeSDK } from '@opentelemetry/sdk-node'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import { Resource } from '@opentelemetry/resources'; +import { + BatchSpanProcessor, + ConsoleSpanExporter, +} from '@opentelemetry/sdk-trace-node'; +import { + BatchLogRecordProcessor, + ConsoleLogRecordExporter, +} from '@opentelemetry/sdk-logs'; +import { + ConsoleMetricExporter, + PeriodicExportingMetricReader, +} from '@opentelemetry/sdk-metrics'; +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'; +import { Config } from '../config/config.js'; +import { SERVICE_NAME, sessionId } from './constants.js'; +import { initializeMetrics } from './metrics.js'; +import { logCliConfiguration } from './loggers.js'; + +// For troubleshooting, set the log level to DiagLogLevel.DEBUG +diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO); + +let sdk: NodeSDK | undefined; +let telemetryInitialized = false; + +export function isTelemetrySdkInitialized(): boolean { + return telemetryInitialized; +} + +function parseGrpcEndpoint( + otlpEndpointSetting: string | undefined, +): string | undefined { + if (!otlpEndpointSetting) { + return undefined; + } + // Trim leading/trailing quotes that might come from env variables + const trimmedEndpoint = otlpEndpointSetting.replace(/^["']|["']$/g, ''); + + try { + const url = new URL(trimmedEndpoint); + // OTLP gRPC exporters expect an endpoint in the format scheme://host:port + // The `origin` property provides this, stripping any path, query, or hash. + return url.origin; + } catch (error) { + diag.error('Invalid OTLP endpoint URL provided:', trimmedEndpoint, error); + return undefined; + } +} + +export function initializeTelemetry(config: Config): void { + if (telemetryInitialized || !config.getTelemetryEnabled()) { + return; + } + + const geminiCliVersion = config.getUserAgent() || 'unknown'; + const resource = new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME, + [SemanticResourceAttributes.SERVICE_VERSION]: geminiCliVersion, + 'session.id': sessionId, + }); + + const otlpEndpoint = config.getTelemetryOtlpEndpoint(); + const grpcParsedEndpoint = parseGrpcEndpoint(otlpEndpoint); + const useOtlp = !!grpcParsedEndpoint; + + const spanExporter = useOtlp + ? new OTLPTraceExporter({ url: grpcParsedEndpoint }) + : new ConsoleSpanExporter(); + const logExporter = useOtlp + ? new OTLPLogExporter({ url: grpcParsedEndpoint }) + : new ConsoleLogRecordExporter(); + const metricReader = useOtlp + ? new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ url: grpcParsedEndpoint }), + exportIntervalMillis: 10000, + }) + : new PeriodicExportingMetricReader({ + exporter: new ConsoleMetricExporter(), + exportIntervalMillis: 10000, + }); + + sdk = new NodeSDK({ + resource, + spanProcessors: [new BatchSpanProcessor(spanExporter)], + logRecordProcessor: new BatchLogRecordProcessor(logExporter), + metricReader, + instrumentations: [new HttpInstrumentation()], + }); + + try { + sdk.start(); + console.log('OpenTelemetry SDK started successfully.'); + telemetryInitialized = true; + initializeMetrics(); + logCliConfiguration(config); + } catch (error) { + console.error('Error starting OpenTelemetry SDK:', error); + } + + process.on('SIGTERM', shutdownTelemetry); + process.on('SIGINT', shutdownTelemetry); +} + +export async function shutdownTelemetry(): Promise<void> { + if (!telemetryInitialized || !sdk) { + return; + } + try { + await sdk.shutdown(); + console.log('OpenTelemetry SDK shut down successfully.'); + } catch (error) { + console.error('Error shutting down SDK:', error); + } finally { + telemetryInitialized = false; + } +} diff --git a/packages/core/src/telemetry/types.ts b/packages/core/src/telemetry/types.ts new file mode 100644 index 00000000..ea65d6de --- /dev/null +++ b/packages/core/src/telemetry/types.ts @@ -0,0 +1,73 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface UserPromptEvent { + 'event.name': 'user_prompt'; + 'event.timestamp': string; // ISO 8601 + prompt_char_count: number; + prompt?: string; +} + +export interface ToolCallEvent { + 'event.name': 'tool_call'; + 'event.timestamp': string; // ISO 8601 + function_name: string; + function_args: Record<string, unknown>; + duration_ms: number; + success: boolean; + error?: string; + error_type?: string; +} + +export interface ApiRequestEvent { + 'event.name': 'api_request'; + 'event.timestamp': string; // ISO 8601 + model: string; + duration_ms: number; + prompt_token_count: number; +} + +export interface ApiErrorEvent { + 'event.name': 'api_error'; + 'event.timestamp': string; // ISO 8601 + model: string; + error: string; + error_type?: string; + status_code?: number | string; + duration_ms: number; + attempt: number; +} + +export interface ApiResponseEvent { + 'event.name': 'api_response'; + 'event.timestamp': string; // ISO 8601 + model: string; + status_code?: number | string; + duration_ms: number; + error?: string; + attempt: number; +} + +export interface CliConfigEvent { + 'event.name': 'cli_config'; + 'event.timestamp': string; // ISO 8601 + model: string; + sandbox_enabled: boolean; + core_tools_enabled: string; + approval_mode: string; + vertex_ai_enabled: boolean; + log_user_prompts_enabled: boolean; + file_filtering_respect_git_ignore: boolean; + file_filtering_allow_build_artifacts: boolean; +} + +export type TelemetryEvent = + | UserPromptEvent + | ToolCallEvent + | ApiRequestEvent + | ApiErrorEvent + | ApiResponseEvent + | CliConfigEvent; |
