Skip to content

Track strings to calculate codeSurvivalRate #2430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hediet merged 4 commits into from
Dec 12, 2025
Merged

Track strings to calculate codeSurvivalRate #2430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b2f8071
Track strings to calculate codeSurvivalRate
Dec 3, 2025
d35f76c
Update test for the new telemetry adding
Dec 3, 2025
efbf353
Sends data to internal telemetry, adds more tests
hediet Dec 5, 2025
ab65a8c
Also report survival rate after 0s and 5s
hediet Dec 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/extension/prompts/node/codeMapper/codeMapperService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,9 @@ function reportEditSurvivalEvent(res: EditSurvivalResult, { requestId, speculati
chatRequestModel,
mapper,
currentFileContent: res.currentFileContent,
textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined,
textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined,
textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined,
}, {
survivalRateFourGram: res.fourGram,
survivalRateNoRevert: res.noRevert,
Expand Down
148 changes: 138 additions & 10 deletions src/extension/test/node/utils.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/

import * as assert from 'assert';
import { suite, test } from 'vitest';
import { expect, suite, test } from 'vitest';
import { EditSurvivalTracker, applyEditsToRanges, compute4GramTextSimilarity } from '../../../platform/editSurvivalTracking/common/editSurvivalTracker';
import { ISerializedStringEdit, StringEdit, StringReplacement } from '../../../util/vs/editor/common/core/edits/stringEdit';
import { OffsetRange } from '../../../util/vs/editor/common/core/ranges/offsetRange';
Expand Down Expand Up @@ -261,16 +261,125 @@ suite('compute4GramTextSimilarity', () => {
});

suite('EditSurvivalTracker', () => {
function renameProps<T extends object>(obj: T, map: { [K in keyof T]?: string }): any {
const result: any = {};
for (const key of Object.keys(obj) as (keyof T)[]) {
const newKey = map[key] || key;
result[newKey] = obj[key];
}
return result;
}

function getScore(input: { text: string; edits: ISerializedStringEdit[] }): unknown {
const originalText = input.text;
const t = new EditSurvivalTracker(originalText, StringEdit.fromJson(input.edits[0]));
t.handleEdits(StringEdit.fromJson(input.edits[1]));
const score = t.computeTrackedEditsSurvivalScore();
return score;
return renameProps(score, {
textBeforeAiEdits: 'text1BeforeAiEdits',
textAfterAiEdits: 'text2AfterAiEdits',
textAfterUserEdits: 'text3AfterUserEdits',
});
}

test('test1', async () => {
assert.deepStrictEqual(
test('simple', async () => {
expect(
getScore(projectableValue_editable({
'text': 'console.log(123456);',
'edits': [
[
{
'pos': 12,
'len': 6,
'txt': `'hello'`
}
],
[
{
'pos': 12,
'len': 7,
'txt': `'Hello'`
}
]
],
'x-editor': 'edit-editor'
})),
).toMatchInlineSnapshot(`
{
"fourGram": 0.5,
"noRevert": 1,
"text1BeforeAiEdits": [
"123456",
],
"text2AfterAiEdits": [
"'hello'",
],
"text3AfterUserEdits": [
"'Hello'",
],
}
`);
});

test('multi edit', async () => {
expect(
getScore(projectableValue_editable({
'text': 'console.log(123456);',
'edits': [
[
{
'pos': 0,
'len': 0,
'txt': '// comment\\n'
},
{
'pos': 12,
'len': 6,
'txt': `'hello'`
}
],
[
{
'pos': 0,
'len': 2,
'txt': '/*'
},
{
'pos': 10,
'len': 2,
'txt': ' */'
},
{
'pos': 25,
'len': 7,
'txt': 'Hello'
}
]
],
'x-editor': 'edit-editor'
})),
).toMatchInlineSnapshot(`
{
"fourGram": 0.4376731301939058,
"noRevert": 1,
"text1BeforeAiEdits": [
"",
"123456",
],
"text2AfterAiEdits": [
"// comment\\n",
"'hello'",
],
"text3AfterUserEdits": [
"/* comment */",
"'Hello",
],
}
`);
});

test('realistic example', async () => {
expect(
getScore(projectableValue_editable({
'text': `import {\r\n\tTextDocument,\r\n\tWebviewPanel,\r\n\tCancellationToken,\r\n\tworkspace,\r\n\tWorkspaceEdit,\r\n\tRange,\r\n\tCustomTextEditorProvider,\r\n} from "vscode";\r\nimport { WebviewInitializer } from "./WebviewInitializer";\r\n\r\ninterface EditableDocument {\r\n\t"x-editable"?: {\r\n\t\tkind: string;\r\n\t\tdefaultUrl: string;\r\n\t};\r\n}\r\n\r\nexport class TextEditorProvider implements CustomTextEditorProvider {\r\n\tconstructor(private readonly webviewInitializer: WebviewInitializer) {}\r\n\r\n\tpublic async resolveCustomTextEditor(\r\n\t\tdocument: TextDocument,\r\n\t\twebviewPanel: WebviewPanel,\r\n\t\ttoken: CancellationToken\r\n\t): Promise<void> {\r\n\t\tlet isThisEditorSaving = false;\r\n\r\n\t\tconst text = document.getText();\r\n\t\tconst doc = JSON.parse(text) as EditableDocument;\r\n\t\tconst args = doc["x-editable"];\r\n\r\n\r\n\t\tconst bridge = this.webviewInitializer.setupWebview(\r\n\t\t\t{ editorUrl: args.defaultUrl },\r\n\t\t\twebviewPanel.webview\r\n\t\t);\r\n\r\n\t\tconst setContentFromDocument = () => {\r\n\t\t\tconst newText = document.getText();\r\n\t\t\tconst content = JSON.parse(newText);\r\n\t\t\tbridge.setContent(content);\r\n\t\t};\r\n\r\n\t\tworkspace.onDidChangeTextDocument(async (evt) => {\r\n\t\t\tif (evt.document !== document) {\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\t\t\tif (isThisEditorSaving) {\r\n\t\t\t\t// We don't want to integrate our own changes\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\t\t\tif (evt.contentChanges.length === 0) {\r\n\t\t\t\t// Sometimes VS Code reports a document change without a change.\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\r\n\t\t\tsetContentFromDocument();\r\n\t\t});\r\n\r\n\t\tbridge.onChange.sub(async ({ newContent }) => {\r\n\t\t\tconst workspaceEdit = new WorkspaceEdit();\r\n\t\t\tconst data = newContent as EditableDocument;\r\n\t\t\tif (!data['x-editable']) {\r\n\t\t\t\tdata['x-editable'] = args;\r\n\t\t\t}\r\n\t\t\tconst output = JSON.stringify(newContent, undefined, 4);\r\n\t\t\tworkspaceEdit.replace(\r\n\t\t\t\tdocument.uri,\r\n\t\t\t\tnew Range(0, 0, document.lineCount, 0),\r\n\t\t\t\toutput\r\n\t\t\t);\r\n\r\n\t\t\tisThisEditorSaving = true;\r\n\t\t\ttry {\r\n\t\t\t\tawait workspace.applyEdit(workspaceEdit);\r\n\t\t\t} finally {\r\n\t\t\t\tisThisEditorSaving = false;\r\n\t\t\t}\r\n\t\t});\r\n\r\n\t\tbridge.onInit.sub(() => {\r\n\t\t\tsetContentFromDocument();\r\n\t\t});\r\n\t}\r\n}\r\n`,
'edits': [
Expand All @@ -290,12 +399,31 @@ suite('EditSurvivalTracker', () => {
]
],
'x-editor': 'edit-editor'
})),
{
'fourGram': 0.75,
'noRevert': 1,
}
);
}))).toMatchInlineSnapshot(
`
{
"fourGram": 0.75,
"noRevert": 1,
"text1BeforeAiEdits": [
"
",
],
"text2AfterAiEdits": [
"

if (!args) {
throw new Error("invalid json document!");
}",
],
"text3AfterUserEdits": [
"

if (!args) {
throw new Error("");
}",
],
}
`);
});
});

Expand Down
13 changes: 13 additions & 0 deletions src/extension/tools/node/abstractReplaceStringTool.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,19 @@ export abstract class AbstractReplaceStringTool<T extends { explanation: string
timeDelayMs: res.timeDelayMs,
didBranchChange: res.didBranchChange ? 1 : 0,
});
res.telemetryService.sendInternalMSFTTelemetryEvent('codeMapper.trackEditSurvival', {
requestId: this._promptContext?.requestId,
requestSource: 'agent',
mapper: 'stringReplaceTool',
textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined,
textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined,
textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined,
}, {
survivalRateFourGram: res.fourGram,
survivalRateNoRevert: res.noRevert,
timeDelayMs: res.timeDelayMs,
didBranchChange: res.didBranchChange ? 1 : 0,
});
res.telemetryService.sendGHTelemetryEvent('replaceString/trackEditSurvival', {
headerRequestId: this._promptContext?.requestId,
requestSource: 'agent',
Expand Down
13 changes: 13 additions & 0 deletions src/extension/tools/node/applyPatchTool.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,6 +399,19 @@ export class ApplyPatchTool implements ICopilotTool<IApplyPatchToolParams> {
timeDelayMs: res.timeDelayMs,
didBranchChange: res.didBranchChange ? 1 : 0,
});
res.telemetryService.sendInternalMSFTTelemetryEvent('applyPatch.trackEditSurvival', {
requestId: this._promptContext?.requestId,
requestSource: 'agent',
mapper: 'applyPatchTool',
textBeforeAiEdits: res.textBeforeAiEdits ? JSON.stringify(res.textBeforeAiEdits) : undefined,
textAfterAiEdits: res.textAfterAiEdits ? JSON.stringify(res.textAfterAiEdits) : undefined,
textAfterUserEdits: res.textAfterUserEdits ? JSON.stringify(res.textAfterUserEdits) : undefined,
}, {
survivalRateFourGram: res.fourGram,
survivalRateNoRevert: res.noRevert,
timeDelayMs: res.timeDelayMs,
didBranchChange: res.didBranchChange ? 1 : 0,
});
res.telemetryService.sendGHTelemetryEvent('applyPatch/trackEditSurvival', {
headerRequestId: this._promptContext?.requestId,
requestSource: 'agent',
Expand Down
13 changes: 12 additions & 1 deletion src/platform/editSurvivalTracking/common/editSurvivalReporter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ export interface EditSurvivalResult {
* See ArcTracker.
*/
readonly arc?: number;

/**
* Text states for each edit region
*/
readonly textBeforeAiEdits?: string[];
readonly textAfterAiEdits?: string[];
readonly textAfterUserEdits?: string[];
Comment on lines +31 to +36
Copy link

Copilot AI Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Consider adding documentation about potential data size implications when serializing textBeforeAiEdits, textAfterAiEdits, and textAfterUserEdits arrays. For files with many edits or large edit regions, the JSON.stringify() calls could produce very large strings in telemetry events. Consider adding a note in the JSDoc about this being intended for internal-only telemetry, or potentially adding size limits/truncation logic if this becomes an issue.

Copilot uses AI. Check for mistakes.
}

export class EditSurvivalReporter {
Expand Down Expand Up @@ -74,6 +81,8 @@ export class EditSurvivalReporter {
this._initialBranchName = this._gitService.activeRepository.get()?.headBranchName;

// This aligns with github inline completions
this._reportAfter(0);
this._reportAfter(5 * 1000);
this._reportAfter(30 * 1000);
this._reportAfter(120 * 1000);
this._reportAfter(300 * 1000);
Expand Down Expand Up @@ -104,7 +113,6 @@ export class EditSurvivalReporter {

const currentBranch = this._getCurrentBranchName();
const didBranchChange = currentBranch !== this._initialBranchName;

this._sendTelemetryEvent({
telemetryService: this._telemetryService,
fourGram: survivalRate.fourGram,
Expand All @@ -113,6 +121,9 @@ export class EditSurvivalReporter {
didBranchChange,
currentFileContent: this._document.getText(),
arc: this._arcTracker?.getAcceptedRestrainedCharactersCount(),
textBeforeAiEdits: survivalRate.textBeforeAiEdits,
textAfterAiEdits: survivalRate.textAfterAiEdits,
textAfterUserEdits: survivalRate.textAfterUserEdits,
});
}

Expand Down
13 changes: 12 additions & 1 deletion src/platform/editSurvivalTracking/common/editSurvivalTracker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,17 @@ export class EditSurvivalTracker {
* fourGram: Number between 0 (no edits survived) and 1 (all edits survived).
* noRevert: Number between 0 (the text after user edits equals the text before the AI edits) and 1 (the text after user edits does not revert any text to the initial state)
Comment on lines 36 to 37
Copy link

Copilot AI Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The JSDoc comment describes the return value as having fourGram and noRevert properties, but the function now also returns textBeforeAiEdits, textAfterAiEdits, and textAfterUserEdits. The documentation should be updated to describe these new fields:

/**
 * Computes survival scores and text snapshots for tracked edits.
 * 
 * @returns An object containing:
 * - fourGram: Number between 0 (no edits survived) and 1 (all edits survived).
 * - noRevert: Number between 0 (the text after user edits equals the text before the AI edits) and 1 (the text after user edits does not revert any text to the initial state)
 * - textBeforeAiEdits: Array of text strings before AI edits were applied for each edit region
 * - textAfterAiEdits: Array of text strings after AI edits were applied for each edit region
 * - textAfterUserEdits: Array of text strings after user edits were applied for each edit region
 */
Suggested change
* fourGram: Number between 0 (no edits survived) and 1 (all edits survived).
* noRevert: Number between 0 (the text after user edits equals the text before the AI edits) and 1 (the text after user edits does not revert any text to the initial state)
* Computes survival scores and text snapshots for tracked edits.
*
* @returns An object containing:
* - fourGram: Number between 0 (no edits survived) and 1 (all edits survived).
* - noRevert: Number between 0 (the text after user edits equals the text before the AI edits) and 1 (the text after user edits does not revert any text to the initial state)
* - textBeforeAiEdits: Array of text strings before AI edits were applied for each edit region.
* - textAfterAiEdits: Array of text strings after AI edits were applied for each edit region.
* - textAfterUserEdits: Array of text strings after user edits were applied for each edit region.

Copilot uses AI. Check for mistakes.
*/
computeTrackedEditsSurvivalScore(): { fourGram: number; noRevert: number } {
computeTrackedEditsSurvivalScore(): { fourGram: number; noRevert: number; textBeforeAiEdits: string[]; textAfterAiEdits: string[]; textAfterUserEdits: string[] } {
let similarityScoreSumFourGram = 0;
let similarityScoreSumMax = 0;

let noRevertSum = 0;
let noRevertSumMax = 0;

const allTextBefore: string[] = [];
const allTextAfter: string[] = [];
const allTextCurrent: string[] = [];

const ranges = this._originalEdits.getNewRanges();
const updatedRanges = applyEditsToRanges(ranges, this._combinedEditsSinceStart);

Expand All @@ -54,6 +58,10 @@ export class EditSurvivalTracker {
const newRange = updatedRanges[i];
const textAfterUserEdits = this._text.substring(newRange.start, newRange.endExclusive);

allTextBefore.push(textBeforeAiEdits);
allTextAfter.push(textAfterAiEdits);
allTextCurrent.push(textAfterUserEdits);

const similarity = compute4GramTextSimilarity(textAfterUserEdits, textAfterAiEdits);

const aiEditSimilarity = compute4GramTextSimilarity(textAfterAiEdits, textBeforeAiEdits);
Expand All @@ -75,6 +83,9 @@ export class EditSurvivalTracker {
return {
fourGram: similarityScoreSumMax === 0 ? 1 : (similarityScoreSumFourGram / similarityScoreSumMax),
noRevert: noRevertSumMax === 0 ? 1 : (noRevertSum / noRevertSumMax),
textBeforeAiEdits: allTextBefore,
textAfterAiEdits: allTextAfter,
textAfterUserEdits: allTextCurrent,
};
}
}
Expand Down