Skip to content

[pr-staging-deploy]: improve reliability of staging deploy lifecycle #417

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rorychatt merged 1 commit into from
Mar 29, 2026
Merged

[pr-staging-deploy]: improve reliability of staging deploy lifecycle #417

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 20 additions & 51 deletions project-demos/pr-staging-deploy/Apps/PrStagingDeployApp.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,20 +293,9 @@ async Task DeployBranchAsync(string branchName, int prNumber, bool clearMessageF
var owner = config["GitHub:Owner"] ?? "";
var repo = config["GitHub:Repo"] ?? "";

if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploying...",
logLines: null,
forceNewComment: true);
}

var result = await deploySvc.DeployBranchAsync(t, branchName, prNumber);
ShowMessage(result.Message, !result.Success);

// Once Sliplane has registered the service (or failed), remove from deployingBranches.
deployingBranches.Set(prev =>
{
var next = new HashSet<string>(prev, StringComparer.OrdinalIgnoreCase);
Expand All @@ -320,14 +309,14 @@ await prComments.TryPostOrUpdateStagingCommentAsync(

if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
// We need at least one service correctly created to start tracking progress.
if (string.IsNullOrEmpty(result.DocsServiceId) && string.IsNullOrEmpty(result.SamplesServiceId))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { "No Sliplane services were created: " + TruncLine(result.Message, 200) });
logLines: new[] { "No Sliplane services were created: " + TruncLine(result.Message, 200) },
forceNewComment: true);
return;
}

Expand All @@ -342,19 +331,14 @@ await commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
return;
}
}
else
else if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repo,
prNumber,
docsUrl: null,
samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(result.Message, 240) });
}
await prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(result.Message, 240) },
forceNewComment: true);
}
}
catch (Exception ex)
Expand All @@ -379,14 +363,6 @@ async Task DeleteBranchAsync(string branchName, int prNumber)
{
var owner = config["GitHub:Owner"] ?? "";
var repo = config["GitHub:Repo"] ?? "";
if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deleting...",
logLines: null);
}

var result = await deploySvc.DeleteBranchAsync(t, prNumber);
ShowMessage(result.Message, !result.Success);
Expand All @@ -396,28 +372,21 @@ await prComments.TryPostOrUpdateStagingCommentAsync(
if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repo,
prNumber,
docsUrl: null,
samplesUrl: null,
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deleted",
logLines: null);
logLines: null,
forceNewComment: true);
}
}
else
else if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
if (!string.IsNullOrEmpty(owner) && !string.IsNullOrEmpty(repo))
{
await prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repo,
prNumber,
docsUrl: null,
samplesUrl: null,
status: "Delete failed",
logLines: new[] { TruncLine(result.Message, 240) });
}
await prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Delete failed",
logLines: new[] { TruncLine(result.Message, 240) },
forceNewComment: true);
}
}
catch (Exception ex) { ShowMessage(ex.Message, true); }
Expand Down
175 changes: 39 additions & 136 deletions project-demos/pr-staging-deploy/Services/GitHubWebhookHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,76 +124,36 @@ private async Task HandlePullRequestAsync(JsonElement root)
"Staging services already exist for PR #{Pr} branch={Branch}, skipping deploy",
prNumber, branch);

// Verify the actual deployment status — services can exist but still be deploying.
var (docsEvOnOpen, samplesEvOnOpen) = await _deployService.GetDeploymentEventsForServicesAsync(
apiToken, existingDepOnOpen.DocsServiceId, existingDepOnOpen.SamplesServiceId);
var statusOnOpen = SliplaneDeploymentStatusResolver.ResolveCombinedStatus(
existingDepOnOpen.DocsServiceId, docsEvOnOpen,
existingDepOnOpen.SamplesServiceId, samplesEvOnOpen);

if (statusOnOpen == "deployed")
// Re-enqueue so the background worker posts the final result when ready.
if (!string.IsNullOrEmpty(existingDepOnOpen.DocsServiceId) || !string.IsNullOrEmpty(existingDepOnOpen.SamplesServiceId))
{
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repoName, prNumber,
docsUrl: existingDepOnOpen.DocsUrl,
samplesUrl: existingDepOnOpen.SamplesUrl,
status: "Already deployed",
forceNewComment: true);
}
else
{
// Still in progress — just update the comment and let the existing
// background worker (if already enqueued) continue monitoring.
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repoName, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploying...",
logLines: null,
forceNewComment: true);

if (!string.IsNullOrEmpty(existingDepOnOpen.DocsServiceId) || !string.IsNullOrEmpty(existingDepOnOpen.SamplesServiceId))
{
await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
Owner: owner,
Repo: repoName,
PrNumber: prNumber,
BranchName: branch,
DocsServiceId: existingDepOnOpen.DocsServiceId,
SamplesServiceId: existingDepOnOpen.SamplesServiceId));
}
await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
Owner: owner,
Repo: repoName,
PrNumber: prNumber,
BranchName: branch,
DocsServiceId: existingDepOnOpen.DocsServiceId,
SamplesServiceId: existingDepOnOpen.SamplesServiceId));
}
break;
}

await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repoName,
prNumber,
docsUrl: null,
samplesUrl: null,
status: "Deploying...",
logLines: null,
forceNewComment: true);

_logger.LogInformation("PR #{Pr} opened: {Title} branch={Branch}", prNumber, title, branch);
var deployResult = await _deployService.DeployBranchAsync(apiToken, branch, prNumber, headRepoCloneUrl);
_logger.LogInformation("Deploy result: {Success} - {Message}", deployResult.Success, deployResult.Message);
if (deployResult.Success)
{
if (string.IsNullOrEmpty(deployResult.DocsServiceId) || string.IsNullOrEmpty(deployResult.SamplesServiceId))
if (string.IsNullOrEmpty(deployResult.DocsServiceId) && string.IsNullOrEmpty(deployResult.SamplesServiceId))
{
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repoName,
prNumber,
docsUrl: null,
samplesUrl: null,
owner, repoName, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(deployResult.Message, 240) });
logLines: new[] { TruncLine(deployResult.Message, 240) },
forceNewComment: true);
break;
}

// Defer "wait until deployed/failed + update links" to background worker.
await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
Owner: owner,
Repo: repoName,
Expand All @@ -205,13 +165,11 @@ await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
else
{
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repoName,
prNumber,
docsUrl: null,
samplesUrl: null,
owner, repoName, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(deployResult.Message, 240) });
logLines: new[] { TruncLine(deployResult.Message, 240) },
forceNewComment: true);
}
break;

Expand All @@ -224,24 +182,13 @@ await _prComments.TryPostOrUpdateStagingCommentAsync(
break;
}

await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repoName,
prNumber,
docsUrl: null,
samplesUrl: null,
status: "Redeploying...",
logLines: null,
forceNewComment: true);

_logger.LogInformation("PR #{Pr} updated: {Branch}", prNumber, branch);
var redeployResult = await _deployService.RedeployBranchAsync(apiToken, branch, prNumber);
_logger.LogInformation("Redeploy result: {Success} - {Message}", redeployResult.Success, redeployResult.Message);

if (!redeployResult.Success)
{
// Services don't exist yet (e.g. first push, or staging was deleted).
// Fall back to a fresh deploy instead of reporting a failure.
// Services don't exist yet — fall back to a fresh deploy.
_logger.LogInformation("PR #{Pr} redeploy found 0 services, falling back to fresh deploy", prNumber);
var fallbackResult = await _deployService.DeployBranchAsync(apiToken, branch, prNumber, headRepoCloneUrl);
_logger.LogInformation("Fallback deploy result: {Success} - {Message}", fallbackResult.Success, fallbackResult.Message);
Expand All @@ -262,7 +209,8 @@ await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repoName, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(fallbackResult.Message, 240) });
logLines: new[] { TruncLine(fallbackResult.Message, 240) },
forceNewComment: true);
}
break;
}
Expand All @@ -274,7 +222,8 @@ await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repoName, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { "Redeploy: docs/samples services not found in Sliplane." });
logLines: new[] { "Redeploy: docs/samples services not found in Sliplane." },
forceNewComment: true);
break;
}

Expand Down Expand Up @@ -361,78 +310,33 @@ private async Task HandleIssueCommentAsync(JsonElement root)
var existingDep = await _deployService.GetDeploymentByPrNumberAsync(apiToken, prNumber);
if (existingDep != null)
{
_logger.LogInformation(
"Staging services already exist for PR #{Pr} branch={Branch}",
prNumber, branch);

// Verify the actual deployment status — services can exist but still be deploying.
var (docsEvents, samplesEvents) = await _deployService.GetDeploymentEventsForServicesAsync(
apiToken, existingDep.DocsServiceId, existingDep.SamplesServiceId);
var currentStatus = SliplaneDeploymentStatusResolver.ResolveCombinedStatus(
existingDep.DocsServiceId, docsEvents,
existingDep.SamplesServiceId, samplesEvents);

if (currentStatus == "deployed")
_logger.LogInformation("Staging services already exist for PR #{Pr}, re-enqueueing monitor", prNumber);
if (!string.IsNullOrEmpty(existingDep.DocsServiceId) || !string.IsNullOrEmpty(existingDep.SamplesServiceId))
{
// Truly deployed — show links.
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: existingDep.DocsUrl,
samplesUrl: existingDep.SamplesUrl,
status: "Already deployed",
forceNewComment: true);
}
else
{
// Deploy is still in progress (or failed) — replace old comment and keep monitoring.
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploying...",
logLines: null,
forceNewComment: true);

// Re-enqueue to keep updating the comment as Sliplane progresses.
if (!string.IsNullOrEmpty(existingDep.DocsServiceId) || !string.IsNullOrEmpty(existingDep.SamplesServiceId))
{
await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
Owner: owner,
Repo: repo,
PrNumber: prNumber,
BranchName: branch,
DocsServiceId: existingDep.DocsServiceId,
SamplesServiceId: existingDep.SamplesServiceId));
}
await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
Owner: owner,
Repo: repo,
PrNumber: prNumber,
BranchName: branch,
DocsServiceId: existingDep.DocsServiceId,
SamplesServiceId: existingDep.SamplesServiceId));
}
return;
}

// Live update: show that deploy started right away.
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repo,
prNumber,
docsUrl: null,
samplesUrl: null,
status: "Deploying...",
logLines: null,
forceNewComment: true);

_logger.LogInformation("PR #{Pr} /deploy comment: {Branch}", prNumber, branch);
var result = await _deployService.DeployBranchAsync(apiToken, branch, prNumber);
_logger.LogInformation("Deploy result: {Success} - {Message}", result.Success, result.Message);
if (result.Success)
{
if (string.IsNullOrEmpty(result.DocsServiceId) || string.IsNullOrEmpty(result.SamplesServiceId))
if (string.IsNullOrEmpty(result.DocsServiceId) && string.IsNullOrEmpty(result.SamplesServiceId))
{
await _prComments.TryPostOrUpdateStagingCommentAsync(
owner,
repo,
prNumber,
docsUrl: null,
samplesUrl: null,
owner, repo, prNumber,
docsUrl: null, samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(result.Message, 240) });
logLines: new[] { TruncLine(result.Message, 240) },
forceNewComment: true);
return;
}

Expand All @@ -443,8 +347,6 @@ await _commentUpdateQueue.EnqueueAsync(new PrStagingDeployCommentUpdateRequest(
BranchName: branch,
DocsServiceId: result.DocsServiceId,
SamplesServiceId: result.SamplesServiceId));

// Final state + links will be posted by background worker.
}
else
{
Expand All @@ -455,7 +357,8 @@ await _prComments.TryPostOrUpdateStagingCommentAsync(
docsUrl: null,
samplesUrl: null,
status: "Deploy failed",
logLines: new[] { TruncLine(result.Message, 240) });
logLines: new[] { TruncLine(result.Message, 240) },
forceNewComment: true);
}
}

Expand Down
Loading
Loading