Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 155 additions & 0 deletions tests/FSharp.Control.AsyncSeq.Benchmarks/AsyncSeqBenchmarks.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
namespace AsyncSeqBenchmarks

open System
open BenchmarkDotNet.Attributes
open BenchmarkDotNet.Configs
open BenchmarkDotNet.Running
open BenchmarkDotNet.Jobs
open BenchmarkDotNet.Engines
open BenchmarkDotNet.Toolchains.InProcess.Emit
open FSharp.Control

/// Core AsyncSeq performance benchmarks focused on foundational operations
[<MemoryDiagnoser>]
[<SimpleJob(RuntimeMoniker.Net80)>]
type AsyncSeqCoreBenchmarks() =

[<Params(1000, 10000)>]
member val ElementCount = 0 with get, set

/// Benchmark unfoldAsync - core sequence generation
[<Benchmark(Baseline = true)>]
member this.UnfoldAsync() =
let generator state = async {
if state < this.ElementCount then
return Some (state, state + 1)
else
return None
}
AsyncSeq.unfoldAsync generator 0
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmark replicate - simple constant generation
[<Benchmark>]
member this.Replicate() =
AsyncSeq.replicate this.ElementCount 42
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmark mapAsync - common transformation
[<Benchmark>]
member this.MapAsync() =
AsyncSeq.replicate this.ElementCount 1
|> AsyncSeq.mapAsync (fun x -> async.Return (x * 2))
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmark chooseAsync with high selectivity
[<Benchmark>]
member this.ChooseAsync() =
AsyncSeq.replicate this.ElementCount 1
|> AsyncSeq.chooseAsync (fun x -> async.Return (Some (x * 2)))
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmarks for append operations (previously had memory leaks)
[<MemoryDiagnoser>]
[<SimpleJob(RuntimeMoniker.Net80)>]
type AsyncSeqAppendBenchmarks() =

[<Params(10, 50, 100)>]
member val ChainCount = 0 with get, set

/// Benchmark chained appends - tests for memory leaks and O(n²) behavior
[<Benchmark>]
member this.ChainedAppends() =
let mutable result = AsyncSeq.singleton 1
for i in 2 .. this.ChainCount do
result <- AsyncSeq.append result (AsyncSeq.singleton i)
result
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmark multiple sequence appends
[<Benchmark>]
member this.MultipleAppends() =
let sequences = [1 .. this.ChainCount] |> List.map (fun i -> AsyncSeq.singleton i)
sequences
|> List.reduce AsyncSeq.append
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmarks for computation builder recursive patterns (previously O(n²))
[<MemoryDiagnoser>]
[<SimpleJob(RuntimeMoniker.Net80)>]
type AsyncSeqBuilderBenchmarks() =

[<Params(50, 100, 200)>]
member val RecursionDepth = 0 with get, set

/// Benchmark recursive asyncSeq computation - tests for O(n²) regression
[<Benchmark>]
member this.RecursiveAsyncSeq() =
let rec generate cnt = asyncSeq {
if cnt = 0 then () else
let! v = async.Return 1
yield v
yield! generate (cnt-1)
}
generate this.RecursionDepth
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Benchmark unfoldAsync equivalent for comparison
[<Benchmark>]
member this.UnfoldAsyncEquivalent() =
AsyncSeq.unfoldAsync (fun cnt -> async {
if cnt = 0 then return None
else
let! v = async.Return 1
return Some (v, cnt - 1)
}) this.RecursionDepth
|> AsyncSeq.iterAsync (fun _ -> async.Return())
|> Async.RunSynchronously

/// Entry point for running benchmarks
module AsyncSeqBenchmarkRunner =

[<EntryPoint>]
let Main args =
printfn "AsyncSeq Performance Benchmarks"
printfn "================================"
printfn "Running comprehensive performance benchmarks to establish baseline metrics"
printfn "and verify fixes for known performance issues (memory leaks, O(n²) patterns)."
printfn ""

let result =
match args |> Array.tryHead with
| Some "core" ->
printfn "Running Core Operations Benchmarks..."
BenchmarkRunner.Run<AsyncSeqCoreBenchmarks>() |> ignore
0
| Some "append" ->
printfn "Running Append Operations Benchmarks..."
BenchmarkRunner.Run<AsyncSeqAppendBenchmarks>() |> ignore
0
| Some "builder" ->
printfn "Running Builder Pattern Benchmarks..."
BenchmarkRunner.Run<AsyncSeqBuilderBenchmarks>() |> ignore
0
| Some "all" | None ->
printfn "Running All Benchmarks..."
BenchmarkRunner.Run<AsyncSeqCoreBenchmarks>() |> ignore
BenchmarkRunner.Run<AsyncSeqAppendBenchmarks>() |> ignore
BenchmarkRunner.Run<AsyncSeqBuilderBenchmarks>() |> ignore
0
| Some suite ->
printfn "Unknown benchmark suite: %s" suite
printfn "Available suites: core, append, builder, all"
1

printfn ""
printfn "Benchmarks completed. Results provide baseline performance metrics"
printfn "for future performance improvements and regression detection."
result
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<IsPackable>false</IsPackable>
<ServerGarbageCollection>true</ServerGarbageCollection>
</PropertyGroup>
<ItemGroup>
<Compile Include="AsyncSeqBenchmarks.fs" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
<ProjectReference Include="..\..\src\FSharp.Control.AsyncSeq\FSharp.Control.AsyncSeq.fsproj" />
</ItemGroup>
</Project>
86 changes: 86 additions & 0 deletions tests/FSharp.Control.AsyncSeq.Benchmarks/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# AsyncSeq Performance Benchmarks

This project contains systematic performance benchmarks for FSharp.Control.AsyncSeq using BenchmarkDotNet.

## Purpose

These benchmarks provide:
- **Baseline Performance Metrics**: Establish current performance characteristics
- **Regression Detection**: Detect performance regressions in future changes
- **Optimization Validation**: Measure impact of performance improvements
- **Issue Verification**: Verify fixes for known performance issues (#35, #57, etc.)

## Benchmark Suites

### Core Operations (`core`)
Tests fundamental AsyncSeq operations:
- `unfoldAsync` - Core sequence generation
- `replicate` - Simple constant generation
- `mapAsync` - Common transformation operation
- `chooseAsync` - Filtering with async projection

### Append Operations (`append`)
Tests append functionality (previously had memory leaks):
- `ChainedAppends` - Sequential append operations
- `MultipleAppends` - Batch append operations

### Builder Patterns (`builder`)
Tests computation expression performance (previously O(n²)):
- `RecursiveAsyncSeq` - Recursive `yield!` patterns
- `UnfoldAsyncEquivalent` - Optimized equivalent using `unfoldAsync`

## Usage

### Build and Run
```bash
# Build benchmarks
dotnet build tests/FSharp.Control.AsyncSeq.Benchmarks -c Release

# Run all benchmark suites
dotnet run --project tests/FSharp.Control.AsyncSeq.Benchmarks -c Release

# Run specific suite
dotnet run --project tests/FSharp.Control.AsyncSeq.Benchmarks -c Release -- core
dotnet run --project tests/FSharp.Control.AsyncSeq.Benchmarks -c Release -- append
dotnet run --project tests/FSharp.Control.AsyncSeq.Benchmarks -c Release -- builder
```

### Interpreting Results

BenchmarkDotNet provides detailed metrics including:
- **Mean**: Average execution time
- **Error**: Standard error of measurements
- **StdDev**: Standard deviation
- **Gen0/Gen1/Gen2**: Garbage collection counts
- **Allocated**: Memory allocations per operation

### Key Performance Indicators

Watch for:
- **Linear scaling**: Execution time should scale linearly with element count
- **Low GC pressure**: Minimal Gen0/Gen1 collections for streaming operations
- **Constant memory**: No memory leaks in long-running operations
- **No O(n²) patterns**: Performance should not degrade quadratically

## Integration with Performance Plan

These benchmarks support the performance improvement plan phases:

**Round 1 (Foundation)**:
- ✅ Systematic benchmarking infrastructure (this project)
- ✅ Baseline performance documentation
- 🔄 Performance regression detection

**Round 2-4**:
- Use these benchmarks to measure optimization impact
- Add new benchmarks for additional performance improvements
- Validate performance targets and success metrics

## Known Issues Being Monitored

- **Issue #35**: Memory leaks in append operations (fixed in PR #193)
- **Issue #57**: O(n²) performance in recursive patterns (fixed in 2016)
- **Issue #50**: Excessive append function calls
- **Issues #74, #95**: Parallelism inefficiencies

Results from these benchmarks help verify these issues remain resolved and detect any regressions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
<None Include="AsyncSeqPerf.fsx" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
<PackageReference Include="coverlet.collector" Version="6.0.0" />
<ProjectReference Include="..\..\src\FSharp.Control.AsyncSeq\FSharp.Control.AsyncSeq.fsproj" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
Expand Down