Nextest

.config/nextest.toml

@@ -0,0 +1,32 @@
+# Note: any test whose name starts with 'e2e' is considered an end-to-end test
+# and will be included or excluded based on the profile used.
+
+nextest-version = "0.9.115"
+
+[profile.default]
+default-filter = "not test(test_e2e)"
+# Skip end-to-end tests by default
+
+[profile.e2e]
+default-filter = "all()"
+
+[profile.ci]
+default-filter = "all()"
+slow-timeout = "20s"
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"
+
+[profile.ci-skip-e2e]
+inherits = "ci"
+default-filter = "not test(test_e2e)"
+
+[[profile.default.overrides]]
+filter = "test(test_e2e)"
+test-group = "e2e"
+retries = { backoff = "fixed", count = 4, delay = "1s" }
+slow-timeout = "60s"
+
+[test-groups.e2e]
+max-threads = 1  # Run end-to-end tests serially

.github/workflows/tests.yml

@@ -20,64 +20,42 @@ on:
   pull_request:
     branches: ["main"]
 
-env:
-  CARGO_TERM_COLOR: always
-  RUST_LOG: debug
-
 jobs:
-  build:
+  test:
+    name: Test (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
         include:
-          # Platform supports is limited for tests (no aarch64)
-          - os: windows-latest
-            target: x86_64-pc-windows-msvc
-          - os: macos-latest
-            target: x86_64-apple-darwin
-            e2e-testing: true
-          - os: ubuntu-latest
-            target: x86_64-unknown-linux-gnu
-            e2e-testing: true
-
-    name: Test (${{ matrix.target }})
-    runs-on: ${{ matrix.os }}
+          - { os: macos-latest, test-profile: ci }
+          - { os: ubuntu-latest, test-profile: ci }
+          - { os: windows-latest, test-profile: ci-skip-e2e }
+    permissions:
+      contents: read
+      actions: read
+      checks: write
+      # Required for test reporting
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
+      - name: Checkout
+        uses: actions/checkout@v3
+        with: { submodules: true }
 
-      - name: Install Rust toolchain
-        run: |
-          rustup update --no-self-update nightly
-          rustup target add ${{ matrix.target }} --toolchain nightly
+      - name: Setup Rust Toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with: { rustflags: "" }  # Don't fail on warnings
+
+      - name: Install Nextest
+        uses: taiki-e/install-action@v2
+        with: { tool: nextest@0.9.115 }
 
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:
           version: "25.2"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Cache Cargo registry
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-          key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-registry-
-
-      - name: Cache Cargo build
-        uses: actions/cache@v3
-        with:
-          path: target/
-          key: ${{ runner.os }}-cargo-build-${{ matrix.target }}-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-build-${{ matrix.target }}-
-            ${{ runner.os }}-cargo-build-
-
       - name: Install linux dependencies
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
@@ -103,7 +81,7 @@ jobs:
             libglib2.0-dev
 
       - name: Install LiveKit server
-        if: ${{ matrix.e2e-testing }}
+        if: ${{ matrix.test-profile == 'ci' }}
         run: |
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
             curl -sSL https://get.livekit.io | bash
@@ -112,17 +90,18 @@ jobs:
           fi
 
       - name: Run LiveKit server
-        if: ${{ matrix.e2e-testing }}
+        if: ${{ matrix.test-profile == 'ci' }}
         run: livekit-server --dev &
 
-      - name: Test (no E2E)
-        if: ${{ !matrix.e2e-testing }}
+      - name: Test (${{ matrix.test-profile }})
         env:
           RUST_LOG: info
-        run: cargo +nightly test --release --verbose --target ${{ matrix.target }} -- --nocapture
+        run: cargo nextest run -P ${{ matrix.test-profile }} --no-capture
 
-      - name: Test (with E2E)
-        if: ${{ matrix.e2e-testing }}
-        env:
-          RUST_LOG: info
-        run: cargo +nightly test --release --verbose --target ${{ matrix.target }} --features __lk-e2e-test -- --nocapture
+      - name: Test Report
+        uses: dorny/test-reporter@v2
+        if: ${{ !cancelled() }}
+        with:
+          name: Test Results
+          path: target/nextest/${{ matrix.test-profile }}/junit.xml
+          reporter: java-junit

livekit/Cargo.toml

@@ -23,7 +23,6 @@ rustls-tls-native-roots = ["livekit-api/rustls-tls-native-roots"]
 rustls-tls-webpki-roots = ["livekit-api/rustls-tls-webpki-roots"]
 __rustls-tls = ["livekit-api/__rustls-tls"]
 __lk-internal = [] # internal features (used by livekit-ffi)
-__lk-e2e-test = [] # end-to-end testing with a LiveKit server
 
 [dependencies]
 livekit-runtime = { workspace = true }

livekit/tests/audio_test.rs

@@ -12,19 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
-use {
-    anyhow::{anyhow, Ok, Result},
-    common::{
-        audio::{ChannelIterExt, FreqAnalyzer, SineParameters, SineTrack},
-        test_rooms,
-    },
-    futures_util::StreamExt,
-    libwebrtc::audio_stream::native::NativeAudioStream,
-    livekit::prelude::*,
-    std::{sync::Arc, time::Duration},
-    tokio::time::timeout,
+use anyhow::{anyhow, Ok, Result};
+use common::{
+    audio::{ChannelIterExt, FreqAnalyzer, SineParameters, SineTrack},
+    test_rooms,
 };
+use futures_util::StreamExt;
+use libwebrtc::audio_stream::native::NativeAudioStream;
+use livekit::prelude::*;
+use std::{sync::Arc, time::Duration};
+use tokio::time::timeout;
 
 mod common;
 
@@ -35,9 +32,8 @@ struct TestParams {
     sub_channels: u32,
 }
 
-#[cfg(feature = "__lk-e2e-test")]
 #[test_log::test(tokio::test)]
-async fn test_audio() -> Result<()> {
+async fn test_e2e_audio() -> Result<()> {
     let test_params = [
         TestParams { pub_rate_hz: 48_000, pub_channels: 1, sub_rate_hz: 48_000, sub_channels: 1 },
         TestParams { pub_rate_hz: 48_000, pub_channels: 2, sub_rate_hz: 48_000, sub_channels: 2 },
@@ -56,7 +52,6 @@ async fn test_audio() -> Result<()> {
 /// Verifies that audio can be published and received correctly
 /// between two participants by detecting the frequency of the sine wave on the subscriber end.
 ///
-#[cfg(feature = "__lk-e2e-test")]
 async fn test_audio_with(params: TestParams) -> Result<()> {
     let mut rooms = test_rooms(2).await?;
     let (pub_room, _) = rooms.pop().unwrap();

livekit/tests/common/mod.rs

@@ -12,9 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
 /// Utilities for end-to-end testing with a LiveKit server.
 mod e2e;
 
-#[cfg(feature = "__lk-e2e-test")]
 pub use e2e::*;

livekit/tests/data_channel_encryption.rs

@@ -12,26 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
-use {
-    crate::common::test_rooms_with_options,
-    anyhow::{Ok, Result},
-    livekit::{
-        e2ee::{
-            key_provider::{KeyProvider, KeyProviderOptions},
-            EncryptionType,
-        },
-        DataPacket, E2eeOptions, RoomEvent, RoomOptions,
+use anyhow::{Ok, Result};
+use common::test_rooms_with_options;
+use livekit::{
+    e2ee::{
+        key_provider::{KeyProvider, KeyProviderOptions},
+        EncryptionType,
     },
-    std::time::Duration,
-    tokio::{time::timeout, try_join},
+    DataPacket, E2eeOptions, RoomEvent, RoomOptions,
 };
+use std::time::Duration;
+use tokio::{time::timeout, try_join};
 
 mod common;
 
-#[cfg(feature = "__lk-e2e-test")]
 #[tokio::test]
-async fn test_data_channel_encryption() -> Result<()> {
+async fn test_e2e_data_channel_encryption() -> Result<()> {
     const ITERATIONS: usize = 128;
     const PAYLOAD_SIZE: usize = 4096;
 

livekit/tests/data_channel_test.rs

@@ -12,23 +12,19 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
-use {
-    crate::common::test_rooms,
-    anyhow::{Ok, Result},
-    livekit::{DataPacket, RoomEvent, SimulateScenario},
-    std::{sync::Arc, time::Duration},
-    tokio::{
-        time::{self, timeout},
-        try_join,
-    },
+use anyhow::{Ok, Result};
+use common::test_rooms;
+use livekit::{DataPacket, RoomEvent, SimulateScenario};
+use std::{sync::Arc, time::Duration};
+use tokio::{
+    time::{self, timeout},
+    try_join,
 };
 
 mod common;
 
-#[cfg(feature = "__lk-e2e-test")]
 #[test_log::test(tokio::test)]
-async fn test_reliable_retry() -> Result<()> {
+async fn test_e2e_reliable_retry() -> Result<()> {
     use anyhow::Context;
 
     const ITERATIONS: usize = 128;

livekit/tests/data_stream_test.rs

@@ -12,21 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
-use {
-    crate::common::test_rooms,
-    anyhow::{anyhow, Ok, Result},
-    chrono::{TimeDelta, Utc},
-    livekit::{RoomEvent, StreamByteOptions, StreamReader, StreamTextOptions},
-    std::time::Duration,
-    tokio::{time::timeout, try_join},
-};
+use anyhow::{anyhow, Ok, Result};
+use chrono::{TimeDelta, Utc};
+use common::test_rooms;
+use livekit::{RoomEvent, StreamByteOptions, StreamReader, StreamTextOptions};
+use std::time::Duration;
+use tokio::{time::timeout, try_join};
 
 mod common;
 
-#[cfg(feature = "__lk-e2e-test")]
 #[tokio::test]
-async fn test_send_bytes() -> Result<()> {
+async fn test_e2e_send_bytes() -> Result<()> {
     let mut rooms = test_rooms(2).await?;
     let (sending_room, _) = rooms.pop().unwrap();
     let (_, mut receiving_event_rx) = rooms.pop().unwrap();
@@ -70,9 +66,8 @@ async fn test_send_bytes() -> Result<()> {
     Ok(())
 }
 
-#[cfg(feature = "__lk-e2e-test")]
 #[tokio::test]
-async fn test_send_text() -> Result<()> {
+async fn test_e2e_send_text() -> Result<()> {
     let mut rooms = test_rooms(2).await?;
     let (sending_room, _) = rooms.pop().unwrap();
     let (_, mut receiving_event_rx) = rooms.pop().unwrap();

livekit/tests/room_test.rs

@@ -12,21 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#[cfg(feature = "__lk-e2e-test")]
-use {
-    anyhow::{Ok, Result},
-    chrono::{TimeDelta, TimeZone, Utc},
-    common::test_rooms,
-    livekit::{ConnectionState, ParticipantKind, RoomEvent},
-    std::time::Duration,
-    tokio::time::{self, timeout},
-};
+use anyhow::{Ok, Result};
+use chrono::{TimeDelta, TimeZone, Utc};
+use common::test_rooms;
+use livekit::{ConnectionState, ParticipantKind, RoomEvent};
+use std::time::Duration;
+use tokio::time::{self, timeout};
 
 mod common;
 
-#[cfg(feature = "__lk-e2e-test")]
 #[test_log::test(tokio::test)]
-async fn test_connect() -> Result<()> {
+async fn test_e2e_connect() -> Result<()> {
     let (room, _) = test_rooms(1).await?.pop().unwrap();
 
     assert_eq!(room.connection_state(), ConnectionState::Connected);
@@ -45,9 +41,8 @@ async fn test_connect() -> Result<()> {
     Ok(())
 }
 
-#[cfg(feature = "__lk-e2e-test")]
 #[test_log::test(tokio::test)]
-async fn test_connect_multiple() -> Result<()> {
+async fn test_e2e_connect_multiple() -> Result<()> {
     let mut rooms = test_rooms(2).await?;
 
     let (second, _) = rooms.pop().unwrap();
@@ -61,9 +56,8 @@ async fn test_connect_multiple() -> Result<()> {
     Ok(())
 }
 
-#[cfg(feature = "__lk-e2e-test")]
 #[test_log::test(tokio::test)]
-async fn test_participant_disconnect() -> Result<()> {
+async fn test_e2e_participant_disconnect() -> Result<()> {
     let mut rooms = test_rooms(2).await?;
     let (second, _) = rooms.pop().unwrap();
     let second_sid = second.local_participant().sid();