fix(tests): refactor integration tests to be less flaky (#4890)

Co-authored-by: matt korwel <[email protected]>

1 files changed, 50 insertions, 6 deletions

diff --git a/integration-tests/replace.test.js b/integration-tests/replace.test.js
index 060aba55..1ac6f5a4 100644
--- a/integration-tests/replace.test.js
+++ b/integration-tests/replace.test.js
@@ -6,17 +6,61 @@
 
 import { test } from 'node:test';
 import { strict as assert } from 'assert';
-import { TestRig } from './test-helper.js';
+import { TestRig, printDebugInfo, validateModelOutput } from './test-helper.js';
 
-test('should be able to replace content in a file', async (t) => {
+test('should be able to replace content in a file', async () => {
   const rig = new TestRig();
-  rig.setup(t.name);
+  await rig.setup('should be able to replace content in a file');
 
   const fileName = 'file_to_replace.txt';
-  rig.createFile(fileName, 'original content');
+  const originalContent = 'original content';
+  const expectedContent = 'replaced content';
+
+  rig.createFile(fileName, originalContent);
   const prompt = `Can you replace 'original' with 'replaced' in the file 'file_to_replace.txt'`;
 
-  await rig.run(prompt);
+  const result = await rig.run(prompt);
+
+  const foundToolCall = await rig.waitForToolCall('replace');
+
+  // Add debugging information
+  if (!foundToolCall) {
+    printDebugInfo(rig, result);
+  }
+
+  assert.ok(foundToolCall, 'Expected to find a replace tool call');
+
+  // Validate model output - will throw if no output, warn if missing expected content
+  validateModelOutput(
+    result,
+    ['replaced', 'file_to_replace.txt'],
+    'Replace content test',
+  );
+
   const newFileContent = rig.readFile(fileName);
-  assert.strictEqual(newFileContent, 'replaced content');
+
+  // Add debugging for file content
+  if (newFileContent !== expectedContent) {
+    console.error('File content mismatch - Debug info:');
+    console.error('Expected:', expectedContent);
+    console.error('Actual:', newFileContent);
+    console.error(
+      'Tool calls:',
+      rig.readToolLogs().map((t) => ({
+        name: t.toolRequest.name,
+        args: t.toolRequest.args,
+      })),
+    );
+  }
+
+  assert.strictEqual(
+    newFileContent,
+    expectedContent,
+    'File content should be updated correctly',
+  );
+
+  // Log success info if verbose
+  if (process.env.VERBOSE === 'true') {
+    console.log('File replaced successfully. New content:', newFileContent);
+  }
 });