Skip to content

Run UI tests with both renderers #877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Kyle-Ye merged 3 commits into from
May 27, 2026
Merged

Run UI tests with both renderers #877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
85f0c6f
Update ViewRendererVendor
Kyle-Ye May 27, 2026
b8fe486
Disable some test case for osui renderer
Kyle-Ye May 27, 2026
e032d57
Run UI tests with both renderers
Kyle-Ye May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 41 additions & 22 deletions .github/workflows/uitests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,23 +115,26 @@ jobs:

outputs["status-enabled"] = "true";
const details_url = `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`;
const renderers = ["SwiftUI Renderer", "OpenSwiftUI Renderer"];
for (const platform of platforms) {
const label = platform === "ios" ? "iOS" : "macOS";
const description = `Queued by @${comment.user.login} with ${body}.`.slice(0, 140);
await github.rest.repos.createCommitStatus({
owner,
repo,
sha: pull.head.sha,
state: "pending",
target_url: details_url,
description,
context: `UI Tests / ${label}`,
});
for (const renderer of renderers) {
const description = `Queued by @${comment.user.login} with ${body}.`.slice(0, 140);
await github.rest.repos.createCommitStatus({
owner,
repo,
sha: pull.head.sha,
state: "pending",
target_url: details_url,
description,
context: `UI Tests / ${label} / ${renderer}`,
});
}
}
setOutputs();

ios_uitest:
name: Execute UI tests on iOS
name: Execute UI tests on iOS (${{ matrix.renderer.name }})
needs: prepare_issue_comment
if: >-
always() && (
Expand All @@ -149,6 +152,13 @@ jobs:
xcode-version: ["16.4"]
release: [2024]
ios-version: ["18.5"]
renderer:
Copy link
Copy Markdown

@augmentcode augmentcode Bot May 27, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With renderer added as a matrix axis, this matrix still relies on an include entry that doesn’t specify renderer; that can produce an extra job where matrix.renderer.* is unset (and/or leave the base combinations without ios-simulator-name). Consider ensuring every generated matrix row defines both ios-simulator-name and renderer.

Severity: high

Fix This in Augment

🤖 Was this useful? React with 👍 or 👎, or 🚀 if it prevented an incident/outage.

- name: SwiftUI Renderer
swiftui_renderer: 1
artifact_suffix: swiftui-renderer
- name: OpenSwiftUI Renderer
swiftui_renderer: 0
artifact_suffix: openswiftui-renderer
include:
- ios-version: "18.5"
ios-simulator-name: "iPhone 16 Pro"
Expand All @@ -157,9 +167,10 @@ jobs:
- self-hosted
- ${{ matrix.os }}
env:
STATUS_CONTEXT: UI Tests / iOS
STATUS_CONTEXT: UI Tests / iOS / ${{ matrix.renderer.name }}
STATUS_ENABLED: ${{ needs.prepare_issue_comment.outputs.status-enabled }}
STATUS_SHA: ${{ needs.prepare_issue_comment.outputs.ref }}
RENDERER_NAME: ${{ matrix.renderer.name }}
OPENSWIFTUI_WERROR: 0 # Disable it to avoid enable OAG's werror and hit conflicts
OPENSWIFTUI_OPENATTRIBUTESHIMS_ATTRIBUTEGRAPH: 1
OPENSWIFTUI_COMPATIBILITY_TEST: 0
Expand All @@ -168,7 +179,7 @@ jobs:
OPENSWIFTUI_TARGET_RELEASE: ${{ matrix.release }}
OPENSWIFTUI_USE_LOCAL_DEPS: 1
OPENSWIFTUI_LINK_TESTING: 0
OPENSWIFTUI_SWIFTUI_RENDERER: 1 # TODO: Add render-off UI test coverage with OPENSWIFTUI_SWIFTUI_RENDERER=0.
OPENSWIFTUI_SWIFTUI_RENDERER: ${{ matrix.renderer.swiftui_renderer }}
GH_TOKEN: ${{ github.token }}
steps:
- name: Mark PR status running
Expand All @@ -183,7 +194,7 @@ jobs:
sha: process.env.STATUS_SHA,
state: "pending",
target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`,
description: "iOS UI tests are running.",
description: `iOS UI tests (${process.env.RENDERER_NAME}) are running.`,
context: process.env.STATUS_CONTEXT,
});
- uses: actions/checkout@v4
Expand All @@ -197,7 +208,7 @@ jobs:
xcode-version: ${{ matrix.xcode-version }}
platform: ios
destination: "platform=iOS Simulator,OS=${{ matrix.ios-version }},name=${{ matrix.ios-simulator-name }}"
artifact-name: ios-uitest-snapshots-${{ matrix.ios-version }}
artifact-name: ios-uitest-snapshots-${{ matrix.ios-version }}-${{ matrix.renderer.artifact_suffix }}
update-reference: ${{ (github.event_name == 'workflow_dispatch' && inputs.update_reference) || needs.prepare_issue_comment.outputs.update-reference == 'true' }}
- name: Fail if tests failed
if: steps.run-tests.outputs.test-result == 'failure'
Expand All @@ -215,7 +226,7 @@ jobs:
"failure";
const description = process.env.JOB_STATUS === "cancelled" ?
"iOS UI tests were cancelled." :
`iOS UI tests completed with ${state}.`;
`iOS UI tests (${process.env.RENDERER_NAME}) completed with ${state}.`;
await github.rest.repos.createCommitStatus({
owner,
repo,
Expand All @@ -227,7 +238,7 @@ jobs:
});

macos_uitest:
name: Execute UI tests on macOS
name: Execute UI tests on macOS (${{ matrix.renderer.name }})
needs: prepare_issue_comment
if: >-
always() && (
Expand All @@ -244,22 +255,30 @@ jobs:
os: [macos-15]
xcode-version: ["16.4"]
release: [2024]
renderer:
- name: SwiftUI Renderer
swiftui_renderer: 1
artifact_suffix: swiftui-renderer
- name: OpenSwiftUI Renderer
swiftui_renderer: 0
artifact_suffix: openswiftui-renderer
# Limit to self-hosted to reduce action cost
runs-on:
- self-hosted
- ${{ matrix.os }}
env:
STATUS_CONTEXT: UI Tests / macOS
STATUS_CONTEXT: UI Tests / macOS / ${{ matrix.renderer.name }}
STATUS_ENABLED: ${{ needs.prepare_issue_comment.outputs.status-enabled }}
STATUS_SHA: ${{ needs.prepare_issue_comment.outputs.ref }}
RENDERER_NAME: ${{ matrix.renderer.name }}
OPENSWIFTUI_WERROR: 0
OPENSWIFTUI_OPENATTRIBUTESHIMS_ATTRIBUTEGRAPH: 1
OPENSWIFTUI_COMPATIBILITY_TEST: 0
OPENSWIFTUI_SWIFT_LOG: 0
OPENSWIFTUI_SWIFT_CRYPTO: 0
OPENSWIFTUI_TARGET_RELEASE: ${{ matrix.release }}
OPENSWIFTUI_USE_LOCAL_DEPS: 1
OPENSWIFTUI_SWIFTUI_RENDERER: 1 # TODO: Add render-off UI test coverage with OPENSWIFTUI_SWIFTUI_RENDERER=0.
OPENSWIFTUI_SWIFTUI_RENDERER: ${{ matrix.renderer.swiftui_renderer }}
GH_TOKEN: ${{ github.token }}
steps:
- name: Mark PR status running
Expand All @@ -274,7 +293,7 @@ jobs:
sha: process.env.STATUS_SHA,
state: "pending",
target_url: `${process.env.GITHUB_SERVER_URL}/${owner}/${repo}/actions/runs/${context.runId}`,
description: "macOS UI tests are running.",
description: `macOS UI tests (${process.env.RENDERER_NAME}) are running.`,
context: process.env.STATUS_CONTEXT,
});
- uses: actions/checkout@v4
Expand All @@ -288,7 +307,7 @@ jobs:
xcode-version: ${{ matrix.xcode-version }}
platform: macos
destination: "platform=macOS"
artifact-name: macos-uitest-snapshots
artifact-name: macos-uitest-snapshots-${{ matrix.renderer.artifact_suffix }}
update-reference: ${{ (github.event_name == 'workflow_dispatch' && inputs.update_reference) || needs.prepare_issue_comment.outputs.update-reference == 'true' }}
- name: Fail if tests failed
if: steps.run-tests.outputs.test-result == 'failure'
Expand All @@ -306,7 +325,7 @@ jobs:
"failure";
const description = process.env.JOB_STATUS === "cancelled" ?
"macOS UI tests were cancelled." :
`macOS UI tests completed with ${state}.`;
`macOS UI tests (${process.env.RENDERER_NAME}) completed with ${state}.`;
await github.rest.repos.createCommitStatus({
owner,
repo,
Expand Down
17 changes: 17 additions & 0 deletions Example/OpenSwiftUIUITests/Export.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,23 @@ let shouldRecord: Bool? = nil
#else
@_exported import SwiftUI
let shouldRecord: Bool? = true

public struct ViewRendererVendor: RawRepresentable, Hashable, CaseIterable {
public let rawValue: String

public init(rawValue: String) {
self.rawValue = rawValue
}

/// OpenSwiftUI's view renderer.
public static let osui = ViewRendererVendor(rawValue: "org.OpenSwiftUIProject.OpenSwiftUI")

/// Apple's SwiftUI view renderer.
public static let sui = ViewRendererVendor(rawValue: "com.apple.SwiftUI")

public static var allCases: [ViewRendererVendor] { [.osui, .sui] }
}
public let viewRendererVendor = ViewRendererVendor.sui
#endif
Copy link
Copy Markdown

@augmentcode augmentcode Bot May 27, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

viewRendererVendor is only defined in this #else compilation branch and is currently hard-coded to .sui; the new .disabled(if: viewRendererVendor == .osui, ...) suite annotations may fail to compile or behave incorrectly when the other branch is active now that CI runs both renderer modes.

Severity: high

Other Locations
  • Example/OpenSwiftUIUITests/Layout/Stack/ZStackIndexUITests.swift:11
  • Example/OpenSwiftUIUITests/Render/GeometryEffect/Rotation3DEffectUITests.swift:10
  • Example/OpenSwiftUIUITests/Render/GeometryEffect/RotationEffectUITests.swift:10
  • Example/OpenSwiftUIUITests/Render/RendererEffect/BlurEffectUITests.swift:10
  • Example/OpenSwiftUIUITests/Render/RendererEffect/ClipEffectUITests.swift:10

Fix This in Augment

🤖 Was this useful? React with 👍 or 👎, or 🚀 if it prevented an incident/outage.

let diffTool: SnapshotTestingConfiguration.DiffTool = .odiff

Expand Down
2 changes: 1 addition & 1 deletion Example/OpenSwiftUIUITests/Layout/Stack/ZStackIndexUITests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import Testing
import SnapshotTesting

@MainActor
@Suite(.snapshots(record: .never, diffTool: diffTool))
@Suite(.snapshots(record: .never, diffTool: diffTool), .disabled(if: viewRendererVendor == .osui, "rotate is not supported"))
struct ZStackIndexUITests {
@Test
func rotateOverlap() {
Expand Down
2 changes: 1 addition & 1 deletion Example/OpenSwiftUIUITests/Render/GeometryEffect/Rotation3DEffectUITests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import Testing
import SnapshotTesting

@MainActor
@Suite(.snapshots(record: .never, diffTool: diffTool))
@Suite(.snapshots(record: .never, diffTool: diffTool), .disabled(if: viewRendererVendor == .osui, "rotate is not supported"))
struct Rotation3DEffectUITests {
@Test
func rotation3DEffect() {
Expand Down
2 changes: 1 addition & 1 deletion Example/OpenSwiftUIUITests/Render/GeometryEffect/RotationEffectUITests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import Testing
import SnapshotTesting

@MainActor
@Suite(.snapshots(record: .never, diffTool: diffTool))
@Suite(.snapshots(record: .never, diffTool: diffTool), .disabled(if: viewRendererVendor == .osui, "rotate is not supported"))
struct RotationEffectUITests {
@Test
func rotationEffect() {
Expand Down
2 changes: 1 addition & 1 deletion Example/OpenSwiftUIUITests/Render/RendererEffect/BlurEffectUITests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import Testing
import SnapshotTesting

@MainActor
@Suite(.snapshots(record: .never, diffTool: diffTool))
@Suite(.snapshots(record: .never, diffTool: diffTool), .disabled(if: viewRendererVendor == .osui, "blur is not supported"))
struct BlurEffectUITests {
@Test
func blurColor() {
Expand Down
2 changes: 1 addition & 1 deletion Example/OpenSwiftUIUITests/Render/RendererEffect/ClipEffectUITests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import Testing
import SnapshotTesting

@MainActor
@Suite(.snapshots(record: .never, diffTool: diffTool))
@Suite(.snapshots(record: .never, diffTool: diffTool), .disabled(if: viewRendererVendor == .osui, "ShapeType is not implemented"))
struct ClipEffectUITests {
// FIXME: Investigate the diff. perceptualPrecision should be 1.0

Expand Down
Loading