Skip to content

Wr/exit codes @W-21513440@ #378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
WillieRuemmele wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Wr/exit codes @W-21513440@ #378

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a49a606
fix: first pass at exit codes
WillieRuemmele Mar 24, 2026
0d9429c
Merge branch 'main' into wr/exitCodes
WillieRuemmele Mar 25, 2026
b6d5eaa
docs: move envs/exit codes to their own help sections
WillieRuemmele Mar 25, 2026
514283d
test: fix NUTs
WillieRuemmele Mar 25, 2026
06d70a6
test: fix NUTs
WillieRuemmele Mar 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions src/commands/agent/test/results.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*/

import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
import { Messages } from '@salesforce/core';
import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core';
import { EnvironmentVariable, Messages } from '@salesforce/core';
import { AgentTester, AgentTestResultsResponse } from '@salesforce/agents';
import { resultFormatFlag, testOutputDirFlag, verboseFlag } from '../../../flags.js';
import { handleTestResults } from '../../../handleTestResults.js';
Expand All @@ -30,6 +30,16 @@ export default class AgentTestResults extends SfCommand<AgentTestResultsResult>
public static readonly description = messages.getMessage('description');
public static readonly examples = messages.getMessages('examples');

public static readonly envVariablesSection = toHelpSection(
'ENVIRONMENT VARIABLES',
EnvironmentVariable.SF_TARGET_ORG
);

public static readonly errorCodes = toHelpSection('ERROR CODES', {
'Succeeded (0)': 'Results retrieved successfully. Test results (passed/failed) are in the output.',
'Failed (1)': "Command couldn't execute due to invalid job ID, API errors, network issues, or system errors.",
});

public static readonly flags = {
'target-org': Flags.requiredOrg(),
'api-version': Flags.orgApiVersion(),
Expand Down
21 changes: 19 additions & 2 deletions src/commands/agent/test/resume.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*/

import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
import { Messages } from '@salesforce/core';
import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core';
import { EnvironmentVariable, Messages } from '@salesforce/core';
import { AgentTester } from '@salesforce/agents';
import { AgentTestCache } from '../../../agentTestCache.js';
import { TestStages } from '../../../testStages.js';
Expand All @@ -30,6 +30,17 @@ export default class AgentTestResume extends SfCommand<AgentTestRunResult> {
public static readonly description = messages.getMessage('description');
public static readonly examples = messages.getMessages('examples');

public static readonly envVariablesSection = toHelpSection(
'ENVIRONMENT VARIABLES',
EnvironmentVariable.SF_TARGET_ORG
);

public static readonly errorCodes = toHelpSection('ERROR CODES', {
'Succeeded (0)': 'Test completed successfully. Test results (passed/failed) are in the JSON output.',
'Failed (1)':
"Command couldn't execute due to invalid job ID, API errors, network issues, or system errors. Exit code 1 also indicates tests encountered execution errors.",
});

public static readonly flags = {
'target-org': Flags.requiredOrg(),
'api-version': Flags.orgApiVersion(),
Expand Down Expand Up @@ -85,6 +96,12 @@ export default class AgentTestResume extends SfCommand<AgentTestRunResult> {
verbose: flags.verbose,
});

// Set exit code to 1 only for execution errors (tests couldn't run properly)
// Test assertion failures are business logic and should not affect exit code
if (response?.testCases.some((tc) => tc.status === 'ERROR')) {
process.exitCode = 1;
}

return { ...response!, runId, status: 'COMPLETED' };
}
}
20 changes: 16 additions & 4 deletions src/commands/agent/test/run-eval.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
*/

import { readFile } from 'node:fs/promises';
import { Flags, SfCommand } from '@salesforce/sf-plugins-core';
import { Messages, Org } from '@salesforce/core';
import { Flags, SfCommand, toHelpSection } from '@salesforce/sf-plugins-core';
import { EnvironmentVariable, Messages, Org } from '@salesforce/core';
import { type EvalPayload, normalizePayload, splitIntoBatches } from '../../../evalNormalizer.js';
import { type EvalApiResponse, formatResults, type ResultFormat } from '../../../evalFormatter.js';
import { resultFormatFlag } from '../../../flags.js';
Expand Down Expand Up @@ -157,6 +157,17 @@ export default class AgentTestRunEval extends SfCommand<RunEvalResult> {
public static state = 'beta';
public static readonly hidden = true;

public static readonly envVariablesSection = toHelpSection(
'ENVIRONMENT VARIABLES',
EnvironmentVariable.SF_TARGET_ORG
);

public static readonly errorCodes = toHelpSection('ERROR CODES', {
'Succeeded (0)': 'Tests completed successfully. Test results (passed/failed) are in the JSON output.',
'Failed (1)':
"Execution error occurred. Tests couldn't run due to API errors, network issues, invalid parameters, or system errors.",
});

public static readonly flags = {
'target-org': Flags.requiredOrg(),
'api-version': Flags.orgApiVersion(),
Expand Down Expand Up @@ -272,8 +283,9 @@ export default class AgentTestRunEval extends SfCommand<RunEvalResult> {
// 10. Build structured result for --json
const { summary, testSummaries } = buildResultSummary(mergedResponse);

// Set exit code to 1 if any tests failed
if (summary.failed > 0 || summary.errors > 0) {
// Set exit code to 1 only for execution errors (tests couldn't run)
// Test failures (assertions failed) are business logic and should not affect exit code
if (summary.errors > 0) {
process.exitCode = 1;
}

Expand Down
21 changes: 19 additions & 2 deletions src/commands/agent/test/run.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* limitations under the License.
*/

import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
import { Messages, SfError } from '@salesforce/core';
import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core';
import { EnvironmentVariable, Messages, SfError } from '@salesforce/core';
import { AgentTester, AgentTestStartResponse } from '@salesforce/agents';
import { colorize } from '@oclif/core/ux';
import { CLIError } from '@oclif/core/errors';
Expand Down Expand Up @@ -62,6 +62,17 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
public static readonly description = messages.getMessage('description');
public static readonly examples = messages.getMessages('examples');

public static readonly envVariablesSection = toHelpSection(
'ENVIRONMENT VARIABLES',
EnvironmentVariable.SF_TARGET_ORG
);

public static readonly errorCodes = toHelpSection('ERROR CODES', {
'Succeeded (0)': 'Test started successfully (without --wait), or test completed successfully (with --wait).',
'Failed (1)':
"Command couldn't execute due to API errors, network issues, invalid test name, or system errors. When using --wait, exit code 1 also indicates tests encountered execution errors.",
});

public static readonly flags = {
'target-org': Flags.requiredOrg(),
'api-version': Flags.orgApiVersion(),
Expand Down Expand Up @@ -129,6 +140,12 @@ export default class AgentTestRun extends SfCommand<AgentTestRunResult> {
verbose: flags.verbose,
});

// Set exit code to 1 only for execution errors (tests couldn't run properly)
// Test assertion failures are business logic and should not affect exit code
if (detailsResponse?.testCases.some((tc) => tc.status === 'ERROR')) {
process.exitCode = 1;
}

return { ...detailsResponse!, status: 'COMPLETED', runId: response.runId };
} else {
this.mso.stop();
Expand Down
17 changes: 15 additions & 2 deletions src/commands/agent/validate/authoring-bundle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { SfCommand, Flags } from '@salesforce/sf-plugins-core';
import { Messages } from '@salesforce/core';
import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core';
import { EnvironmentVariable, Messages } from '@salesforce/core';
import { MultiStageOutput } from '@oclif/multi-stage-output';
import { Agent } from '@salesforce/agents';
import { colorize } from '@oclif/core/ux';
Expand All @@ -35,6 +35,19 @@ export default class AgentValidateAuthoringBundle extends SfCommand<AgentValidat
public static readonly examples = messages.getMessages('examples');
public static readonly requiresProject = true;

public static readonly envVariablesSection = toHelpSection(
'ENVIRONMENT VARIABLES',
EnvironmentVariable.SF_TARGET_ORG
);

public static readonly errorCodes = toHelpSection('ERROR CODES', {
'Succeeded (0)': 'Agent Script file compiled successfully without errors.',
'Failed (1)': 'Compilation errors found in the Agent Script file.',
'NotFound (2)':
'Validation/compilation API returned HTTP 404. The API endpoint may not be available in your org or region.',
'ServerError (3)': 'Validation/compilation API returned HTTP 500. A server error occurred during compilation.',
});

public static readonly flags = {
'target-org': Flags.requiredOrg(),
'api-version': Flags.orgApiVersion(),
Expand Down
109 changes: 50 additions & 59 deletions test/nuts/agent.test.run-eval.nut.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,105 +39,98 @@ describe('agent test run-eval', function () {
describe('run-eval with JSON file', () => {
it('should run evaluation with JSON payload file', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;

expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(output?.result.tests.length).to.be.greaterThan(0);
expect(output?.result.summary).to.be.ok;
expect(output?.result.summary.passed).to.be.a('number');
expect(output?.result.summary.failed).to.be.a('number');
expect(output?.result.summary.scored).to.be.a('number');
expect(output?.result.summary.errors).to.be.a('number');
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0);
expect(result.jsonOutput?.result.summary).to.be.ok;
expect(result.jsonOutput?.result.summary.passed).to.be.a('number');
expect(result.jsonOutput?.result.summary.failed).to.be.a('number');
expect(result.jsonOutput?.result.summary.scored).to.be.a('number');
expect(result.jsonOutput?.result.summary.errors).to.be.a('number');
});

it('should run evaluation with normalized payload', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

expect(output?.result.tests[0]).to.be.ok;
expect(output?.result.tests[0].id).to.equal('test-topic-routing');
expect(output?.result.tests[0].status).to.be.oneOf(['passed', 'failed']);
expect(output?.result.tests[0].evaluations).to.be.an('array');
expect(result.jsonOutput?.result.tests[0]).to.be.ok;
expect(result.jsonOutput?.result.tests[0].id).to.equal('test-topic-routing');
expect(result.jsonOutput?.result.tests[0].status).to.be.oneOf(['passed', 'failed']);
expect(result.jsonOutput?.result.tests[0].evaluations).to.be.an('array');
});
});

describe('run-eval with YAML file', () => {
it('should run evaluation with YAML test spec file', async () => {
const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(output?.result.tests.length).to.be.greaterThan(0);
expect(output?.result.summary).to.be.ok;
expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0);
expect(result.jsonOutput?.result.summary).to.be.ok;
});

it('should auto-infer agent name from YAML subjectName', async () => {
const command = `agent test run-eval --spec ${yamlSpecPath} --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

// Should succeed without explicit --api-name flag
expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
});

it('should handle YAML spec with contextVariables', async () => {
const command = `agent test run-eval --spec ${yamlWithContextPath} --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

// Verify the command succeeds with contextVariables
expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(output?.result.tests.length).to.be.greaterThan(0);
expect(output?.result.summary).to.be.ok;
expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result.tests.length).to.be.greaterThan(0);
expect(result.jsonOutput?.result.summary).to.be.ok;
});
});

describe('run-eval with flags', () => {
it('should respect --no-normalize flag', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --no-normalize --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 1 });

expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
});

it('should use custom batch size', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --batch-size 1 --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

expect(output?.result).to.be.ok;
expect(output?.result.tests).to.be.an('array');
expect(result.jsonOutput?.result).to.be.ok;
expect(result.jsonOutput?.result.tests).to.be.an('array');
});

it('should support different result formats', async () => {
// Test human format (default) - don't enforce exit code since tests may fail
// Test human format (default)
const humanCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format human --target-org ${getUsername()}`;
const humanOutput = execCmd(humanCommand).shellOutput.stdout;
const humanResult = execCmd(humanCommand, { ensureExitCode: 0 });

expect(humanOutput).to.be.ok;
expect(humanOutput).to.be.a('string');
expect(humanResult.shellOutput.stdout).to.be.ok;
expect(humanResult.shellOutput.stdout).to.be.a('string');

// Test tap format - don't enforce exit code since tests may fail
// Test tap format
const tapCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format tap --target-org ${getUsername()}`;
const tapOutput = execCmd(tapCommand).shellOutput.stdout;
const tapResult = execCmd(tapCommand, { ensureExitCode: 0 });

expect(tapOutput).to.include('TAP version');
expect(tapResult.shellOutput.stdout).to.include('TAP version');

// Test junit format - don't enforce exit code since tests may fail
// Test junit format
const junitCommand = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --result-format junit --target-org ${getUsername()}`;
const junitOutput = execCmd(junitCommand).shellOutput.stdout;
const junitResult = execCmd(junitCommand, { ensureExitCode: 0 });

expect(junitOutput).to.include('<?xml');
expect(junitOutput).to.include('testsuite');
expect(junitResult.shellOutput.stdout).to.include('<?xml');
expect(junitResult.shellOutput.stdout).to.include('testsuite');
});
});

Expand Down Expand Up @@ -183,11 +176,10 @@ describe('agent test run-eval', function () {
describe('run-eval output structure', () => {
it('should include test summaries with correct structure', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

expect(output?.result.tests).to.be.an('array');
const firstTest = output?.result.tests[0];
expect(result.jsonOutput?.result.tests).to.be.an('array');
const firstTest = result.jsonOutput?.result.tests[0];
expect(firstTest).to.have.property('id');
expect(firstTest).to.have.property('status');
expect(firstTest).to.have.property('evaluations');
Expand All @@ -198,10 +190,9 @@ describe('agent test run-eval', function () {

it('should include summary with all metrics', async () => {
const command = `agent test run-eval --spec ${jsonPayloadPath} --api-name Local_Info_Agent --target-org ${getUsername()} --json`;
// Don't enforce exit code 0 since the command exits with 1 if tests fail
const output = execCmd<RunEvalResult>(command).jsonOutput;
const result = execCmd<RunEvalResult>(command, { ensureExitCode: 0 });

const summary = output?.result.summary;
const summary = result.jsonOutput?.result.summary;
expect(summary).to.have.property('passed');
expect(summary).to.have.property('failed');
expect(summary).to.have.property('scored');
Expand Down
Loading