summaryrefslogtreecommitdiff
path: root/packages/core
diff options
context:
space:
mode:
authorJerop Kipruto <[email protected]>2025-06-15 16:24:53 -0400
committerGitHub <[email protected]>2025-06-15 13:24:53 -0700
commit714421c2da4f5d6b9c1c7060fdf5c47ba1c965ca (patch)
tree488459669757dda99eda46b621a030ece38842dd /packages/core
parent4421ef126fc6a2de89132aa35c261bf78cd481d2 (diff)
Add file operation telemetry (#1068)
Introduces telemetry for file create, read, and update operations. This change adds the `gemini_cli.file.operation.count` metric, recorded by the `read-file`, `read-many-files`, and `write-file` tools. The metric includes the following attributes: - `operation` (string: `create`, `read`, `update`): The type of file operation. - `lines` (optional, Int): Number of lines in the file. - `mimetype` (optional, string): Mimetype of the file. - `extension` (optional, string): File extension of the file. Here is a stacked bar chart of file operations by extension (`js`, `ts`, `md`): ![image](https://github.com/user-attachments/assets/3e8f8ea9-6155-4186-863c-075cc47647c5) Here is a stacked bar chart of file operations by type (`create`, `read`, `update`): ![image](https://github.com/user-attachments/assets/3fcf491d-31d0-4ba8-80e6-7fd2bd9c7c27) #750 cc @allenhutchison as discussed
Diffstat (limited to 'packages/core')
-rw-r--r--packages/core/src/telemetry/constants.ts1
-rw-r--r--packages/core/src/telemetry/metrics.test.ts198
-rw-r--r--packages/core/src/telemetry/metrics.ts31
-rw-r--r--packages/core/src/tools/read-file.ts18
-rw-r--r--packages/core/src/tools/read-many-files.ts17
-rw-r--r--packages/core/src/tools/write-file.ts26
-rw-r--r--packages/core/src/utils/fileUtils.ts10
7 files changed, 265 insertions, 36 deletions
diff --git a/packages/core/src/telemetry/constants.ts b/packages/core/src/telemetry/constants.ts
index bbcdbada..de760205 100644
--- a/packages/core/src/telemetry/constants.ts
+++ b/packages/core/src/telemetry/constants.ts
@@ -19,3 +19,4 @@ export const METRIC_API_REQUEST_COUNT = 'gemini_cli.api.request.count';
export const METRIC_API_REQUEST_LATENCY = 'gemini_cli.api.request.latency';
export const METRIC_TOKEN_USAGE = 'gemini_cli.token.usage';
export const METRIC_SESSION_COUNT = 'gemini_cli.session.count';
+export const METRIC_FILE_OPERATION_COUNT = 'gemini_cli.file.operation.count';
diff --git a/packages/core/src/telemetry/metrics.test.ts b/packages/core/src/telemetry/metrics.test.ts
index 7e24b9ad..4fcdd9e1 100644
--- a/packages/core/src/telemetry/metrics.test.ts
+++ b/packages/core/src/telemetry/metrics.test.ts
@@ -5,32 +5,84 @@
*/
import { describe, it, expect, vi, beforeEach, type Mock } from 'vitest';
-import { Counter, Meter, metrics } from '@opentelemetry/api';
-import { initializeMetrics, recordTokenUsageMetrics } from './metrics.js';
+import type {
+ Counter,
+ Meter,
+ Attributes,
+ Context,
+ Histogram,
+} from '@opentelemetry/api';
import { Config } from '../config/config.js';
+import { FileOperation } from './metrics.js';
-const mockCounter = {
- add: vi.fn(),
+const mockCounterAddFn: Mock<
+ (value: number, attributes?: Attributes, context?: Context) => void
+> = vi.fn();
+const mockHistogramRecordFn: Mock<
+ (value: number, attributes?: Attributes, context?: Context) => void
+> = vi.fn();
+
+const mockCreateCounterFn: Mock<(name: string, options?: unknown) => Counter> =
+ vi.fn();
+const mockCreateHistogramFn: Mock<
+ (name: string, options?: unknown) => Histogram
+> = vi.fn();
+
+const mockCounterInstance = {
+ add: mockCounterAddFn,
} as unknown as Counter;
-const mockMeter = {
- createCounter: vi.fn().mockReturnValue(mockCounter),
- createHistogram: vi.fn().mockReturnValue({ record: vi.fn() }),
+const mockHistogramInstance = {
+ record: mockHistogramRecordFn,
+} as unknown as Histogram;
+
+const mockMeterInstance = {
+ createCounter: mockCreateCounterFn.mockReturnValue(mockCounterInstance),
+ createHistogram: mockCreateHistogramFn.mockReturnValue(mockHistogramInstance),
} as unknown as Meter;
-vi.mock('@opentelemetry/api', () => ({
- metrics: {
- getMeter: vi.fn(),
- },
- ValueType: {
- INT: 1,
- },
-}));
+function originalOtelMockFactory() {
+ return {
+ metrics: {
+ getMeter: vi.fn(),
+ },
+ ValueType: {
+ INT: 1,
+ },
+ };
+}
+
+vi.mock('@opentelemetry/api', originalOtelMockFactory);
describe('Telemetry Metrics', () => {
- beforeEach(() => {
- vi.clearAllMocks();
- (metrics.getMeter as Mock).mockReturnValue(mockMeter);
+ let initializeMetricsModule: typeof import('./metrics.js').initializeMetrics;
+ let recordTokenUsageMetricsModule: typeof import('./metrics.js').recordTokenUsageMetrics;
+ let recordFileOperationMetricModule: typeof import('./metrics.js').recordFileOperationMetric;
+
+ beforeEach(async () => {
+ vi.resetModules();
+ vi.doMock('@opentelemetry/api', () => {
+ const actualApi = originalOtelMockFactory();
+ (actualApi.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance);
+ return actualApi;
+ });
+
+ const metricsJsModule = await import('./metrics.js');
+ initializeMetricsModule = metricsJsModule.initializeMetrics;
+ recordTokenUsageMetricsModule = metricsJsModule.recordTokenUsageMetrics;
+ recordFileOperationMetricModule = metricsJsModule.recordFileOperationMetric;
+
+ const otelApiModule = await import('@opentelemetry/api');
+
+ mockCounterAddFn.mockClear();
+ mockCreateCounterFn.mockClear();
+ mockCreateHistogramFn.mockClear();
+ mockHistogramRecordFn.mockClear();
+ (otelApiModule.metrics.getMeter as Mock).mockClear();
+
+ (otelApiModule.metrics.getMeter as Mock).mockReturnValue(mockMeterInstance);
+ mockCreateCounterFn.mockReturnValue(mockCounterInstance);
+ mockCreateHistogramFn.mockReturnValue(mockHistogramInstance);
});
describe('recordTokenUsageMetrics', () => {
@@ -39,14 +91,18 @@ describe('Telemetry Metrics', () => {
} as unknown as Config;
it('should not record metrics if not initialized', () => {
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 100, 'input');
- expect(mockCounter.add).not.toHaveBeenCalled();
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
+ expect(mockCounterAddFn).not.toHaveBeenCalled();
});
it('should record token usage with the correct attributes', () => {
- initializeMetrics(mockConfig);
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 100, 'input');
- expect(mockCounter.add).toHaveBeenCalledWith(100, {
+ initializeMetricsModule(mockConfig);
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 100, 'input');
+ expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
+ expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
+ 'session.id': 'test-session-id',
+ });
+ expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 100, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'input',
@@ -54,30 +110,32 @@ describe('Telemetry Metrics', () => {
});
it('should record token usage for different types', () => {
- initializeMetrics(mockConfig);
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 50, 'output');
- expect(mockCounter.add).toHaveBeenCalledWith(50, {
+ initializeMetricsModule(mockConfig);
+ mockCounterAddFn.mockClear();
+
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 50, 'output');
+ expect(mockCounterAddFn).toHaveBeenCalledWith(50, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'output',
});
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 25, 'thought');
- expect(mockCounter.add).toHaveBeenCalledWith(25, {
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 25, 'thought');
+ expect(mockCounterAddFn).toHaveBeenCalledWith(25, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'thought',
});
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 75, 'cache');
- expect(mockCounter.add).toHaveBeenCalledWith(75, {
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 75, 'cache');
+ expect(mockCounterAddFn).toHaveBeenCalledWith(75, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'cache',
});
- recordTokenUsageMetrics(mockConfig, 'gemini-pro', 125, 'tool');
- expect(mockCounter.add).toHaveBeenCalledWith(125, {
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-pro', 125, 'tool');
+ expect(mockCounterAddFn).toHaveBeenCalledWith(125, {
'session.id': 'test-session-id',
model: 'gemini-pro',
type: 'tool',
@@ -85,13 +143,83 @@ describe('Telemetry Metrics', () => {
});
it('should handle different models', () => {
- initializeMetrics(mockConfig);
- recordTokenUsageMetrics(mockConfig, 'gemini-ultra', 200, 'input');
- expect(mockCounter.add).toHaveBeenCalledWith(200, {
+ initializeMetricsModule(mockConfig);
+ mockCounterAddFn.mockClear();
+
+ recordTokenUsageMetricsModule(mockConfig, 'gemini-ultra', 200, 'input');
+ expect(mockCounterAddFn).toHaveBeenCalledWith(200, {
'session.id': 'test-session-id',
model: 'gemini-ultra',
type: 'input',
});
});
});
+
+ describe('recordFileOperationMetric', () => {
+ const mockConfig = {
+ getSessionId: () => 'test-session-id',
+ } as unknown as Config;
+
+ it('should not record metrics if not initialized', () => {
+ recordFileOperationMetricModule(
+ mockConfig,
+ FileOperation.CREATE,
+ 10,
+ 'text/plain',
+ 'txt',
+ );
+ expect(mockCounterAddFn).not.toHaveBeenCalled();
+ });
+
+ it('should record file creation with all attributes', () => {
+ initializeMetricsModule(mockConfig);
+ recordFileOperationMetricModule(
+ mockConfig,
+ FileOperation.CREATE,
+ 10,
+ 'text/plain',
+ 'txt',
+ );
+
+ expect(mockCounterAddFn).toHaveBeenCalledTimes(2);
+ expect(mockCounterAddFn).toHaveBeenNthCalledWith(1, 1, {
+ 'session.id': 'test-session-id',
+ });
+ expect(mockCounterAddFn).toHaveBeenNthCalledWith(2, 1, {
+ 'session.id': 'test-session-id',
+ operation: FileOperation.CREATE,
+ lines: 10,
+ mimetype: 'text/plain',
+ extension: 'txt',
+ });
+ });
+
+ it('should record file read with minimal attributes', () => {
+ initializeMetricsModule(mockConfig);
+ mockCounterAddFn.mockClear();
+
+ recordFileOperationMetricModule(mockConfig, FileOperation.READ);
+ expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+ 'session.id': 'test-session-id',
+ operation: FileOperation.READ,
+ });
+ });
+
+ it('should record file update with some attributes', () => {
+ initializeMetricsModule(mockConfig);
+ mockCounterAddFn.mockClear();
+
+ recordFileOperationMetricModule(
+ mockConfig,
+ FileOperation.UPDATE,
+ undefined,
+ 'application/javascript',
+ );
+ expect(mockCounterAddFn).toHaveBeenCalledWith(1, {
+ 'session.id': 'test-session-id',
+ operation: FileOperation.UPDATE,
+ mimetype: 'application/javascript',
+ });
+ });
+ });
});
diff --git a/packages/core/src/telemetry/metrics.ts b/packages/core/src/telemetry/metrics.ts
index 59979ef3..124bc602 100644
--- a/packages/core/src/telemetry/metrics.ts
+++ b/packages/core/src/telemetry/metrics.ts
@@ -20,15 +20,23 @@ import {
METRIC_API_REQUEST_LATENCY,
METRIC_TOKEN_USAGE,
METRIC_SESSION_COUNT,
+ METRIC_FILE_OPERATION_COUNT,
} from './constants.js';
import { Config } from '../config/config.js';
+export enum FileOperation {
+ CREATE = 'create',
+ READ = 'read',
+ UPDATE = 'update',
+}
+
let cliMeter: Meter | undefined;
let toolCallCounter: Counter | undefined;
let toolCallLatencyHistogram: Histogram | undefined;
let apiRequestCounter: Counter | undefined;
let apiRequestLatencyHistogram: Histogram | undefined;
let tokenUsageCounter: Counter | undefined;
+let fileOperationCounter: Counter | undefined;
let isMetricsInitialized = false;
function getCommonAttributes(config: Config): Attributes {
@@ -75,7 +83,10 @@ export function initializeMetrics(config: Config): void {
description: 'Counts the total number of tokens used.',
valueType: ValueType.INT,
});
-
+ fileOperationCounter = meter.createCounter(METRIC_FILE_OPERATION_COUNT, {
+ description: 'Counts file operations (create, read, update).',
+ valueType: ValueType.INT,
+ });
const sessionCounter = meter.createCounter(METRIC_SESSION_COUNT, {
description: 'Count of CLI sessions started.',
valueType: ValueType.INT,
@@ -171,3 +182,21 @@ export function recordApiErrorMetrics(
model,
});
}
+
+export function recordFileOperationMetric(
+ config: Config,
+ operation: FileOperation,
+ lines?: number,
+ mimetype?: string,
+ extension?: string,
+): void {
+ if (!fileOperationCounter || !isMetricsInitialized) return;
+ const attributes: Attributes = {
+ ...getCommonAttributes(config),
+ operation,
+ };
+ if (lines !== undefined) attributes.lines = lines;
+ if (mimetype !== undefined) attributes.mimetype = mimetype;
+ if (extension !== undefined) attributes.extension = extension;
+ fileOperationCounter.add(1, attributes);
+}
diff --git a/packages/core/src/tools/read-file.ts b/packages/core/src/tools/read-file.ts
index 586a7123..5cf49209 100644
--- a/packages/core/src/tools/read-file.ts
+++ b/packages/core/src/tools/read-file.ts
@@ -10,6 +10,11 @@ import { makeRelative, shortenPath } from '../utils/paths.js';
import { BaseTool, ToolResult } from './tools.js';
import { isWithinRoot, processSingleFileContent } from '../utils/fileUtils.js';
import { Config } from '../config/config.js';
+import { getSpecificMimeType } from '../utils/fileUtils.js';
+import {
+ recordFileOperationMetric,
+ FileOperation,
+} from '../telemetry/metrics.js';
/**
* Parameters for the ReadFile tool
@@ -145,6 +150,19 @@ export class ReadFileTool extends BaseTool<ReadFileToolParams, ToolResult> {
};
}
+ const lines =
+ typeof result.llmContent === 'string'
+ ? result.llmContent.split('\n').length
+ : undefined;
+ const mimetype = getSpecificMimeType(params.absolute_path);
+ recordFileOperationMetric(
+ this.config,
+ FileOperation.READ,
+ lines,
+ mimetype,
+ path.extname(params.absolute_path),
+ );
+
return {
llmContent: result.llmContent,
returnDisplay: result.returnDisplay,
diff --git a/packages/core/src/tools/read-many-files.ts b/packages/core/src/tools/read-many-files.ts
index 107e16b3..62430c10 100644
--- a/packages/core/src/tools/read-many-files.ts
+++ b/packages/core/src/tools/read-many-files.ts
@@ -14,9 +14,14 @@ import {
detectFileType,
processSingleFileContent,
DEFAULT_ENCODING,
+ getSpecificMimeType,
} from '../utils/fileUtils.js';
import { PartListUnion } from '@google/genai';
import { Config } from '../config/config.js';
+import {
+ recordFileOperationMetric,
+ FileOperation,
+} from '../telemetry/metrics.js';
/**
* Parameters for the ReadManyFilesTool.
@@ -420,6 +425,18 @@ Use this tool when the user's query implies needing the content of several files
contentParts.push(fileReadResult.llmContent); // This is a Part for image/pdf
}
processedFilesRelativePaths.push(relativePathForDisplay);
+ const lines =
+ typeof fileReadResult.llmContent === 'string'
+ ? fileReadResult.llmContent.split('\n').length
+ : undefined;
+ const mimetype = getSpecificMimeType(filePath);
+ recordFileOperationMetric(
+ this.config,
+ FileOperation.READ,
+ lines,
+ mimetype,
+ path.extname(filePath),
+ );
}
}
diff --git a/packages/core/src/tools/write-file.ts b/packages/core/src/tools/write-file.ts
index b9e07034..b19b00ac 100644
--- a/packages/core/src/tools/write-file.ts
+++ b/packages/core/src/tools/write-file.ts
@@ -26,6 +26,11 @@ import {
import { GeminiClient } from '../core/client.js';
import { DEFAULT_DIFF_OPTIONS } from './diffOptions.js';
import { ModifiableTool, ModifyContext } from './modifiable-tool.js';
+import { getSpecificMimeType } from '../utils/fileUtils.js';
+import {
+ recordFileOperationMetric,
+ FileOperation,
+} from '../telemetry/metrics.js';
/**
* Parameters for the WriteFile tool
@@ -271,6 +276,27 @@ export class WriteFileTool
const displayResult: FileDiff = { fileDiff, fileName };
+ const lines = fileContent.split('\n').length;
+ const mimetype = getSpecificMimeType(params.file_path);
+ const extension = path.extname(params.file_path); // Get extension
+ if (isNewFile) {
+ recordFileOperationMetric(
+ this.config,
+ FileOperation.CREATE,
+ lines,
+ mimetype,
+ extension,
+ );
+ } else {
+ recordFileOperationMetric(
+ this.config,
+ FileOperation.UPDATE,
+ lines,
+ mimetype,
+ extension,
+ );
+ }
+
return {
llmContent: llmSuccessMessage,
returnDisplay: displayResult,
diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts
index d726c053..cb4d333a 100644
--- a/packages/core/src/utils/fileUtils.ts
+++ b/packages/core/src/utils/fileUtils.ts
@@ -17,6 +17,16 @@ const MAX_LINE_LENGTH_TEXT_FILE = 2000;
export const DEFAULT_ENCODING: BufferEncoding = 'utf-8';
/**
+ * Looks up the specific MIME type for a file path.
+ * @param filePath Path to the file.
+ * @returns The specific MIME type string (e.g., 'text/python', 'application/javascript') or undefined if not found or ambiguous.
+ */
+export function getSpecificMimeType(filePath: string): string | undefined {
+ const lookedUpMime = mime.lookup(filePath);
+ return typeof lookedUpMime === 'string' ? lookedUpMime : undefined;
+}
+
+/**
* Checks if a path is within a given root directory.
* @param pathToCheck The absolute path to check.
* @param rootDirectory The absolute root directory.