[SPIKE] Adding all implementations for SqlServer, MySql and Postgresql persistence

src/Directory.Packages.props

@@ -19,6 +19,10 @@
     <PackageVersion Include="HdrHistogram" Version="2.5.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.Testing" Version="8.0.21" />
     <PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.21" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore" Version="8.0.11" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.11" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="8.0.11" />
+    <PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="8.0.11" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
     <PackageVersion Include="Microsoft.Extensions.DependencyModel" Version="8.0.2" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
@@ -50,6 +54,7 @@
     <PackageVersion Include="NServiceBus.Transport.Msmq.Sources" Version="3.0.2" />
     <PackageVersion Include="NServiceBus.Transport.SqlServer" Version="8.1.9" />
     <PackageVersion Include="NServiceBus.Transport.PostgreSql" Version="8.1.9" />
+    <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.10" />
     <PackageVersion Include="NuGet.Versioning" Version="6.14.0" />
     <PackageVersion Include="NUnit" Version="4.4.0" />
     <PackageVersion Include="NUnit.Analyzers" Version="4.11.2" />
@@ -62,6 +67,7 @@
     <PackageVersion Include="Particular.LicensingComponent.Report" Version="1.0.0" />
     <PackageVersion Include="Particular.Obsoletes" Version="1.0.0" />
     <PackageVersion Include="Polly.Core" Version="8.5.2" />
+    <PackageVersion Include="Pomelo.EntityFrameworkCore.MySql" Version="8.0.2" />
     <PackageVersion Include="PropertyChanged.Fody" Version="4.1.0" />
     <PackageVersion Include="PropertyChanging.Fody" Version="1.30.3" />
     <PackageVersion Include="PublicApiGenerator" Version="11.5.4" />
@@ -78,6 +84,9 @@
     <PackageVersion Include="System.Reactive.Linq" Version="6.0.1" />
     <PackageVersion Include="System.Reflection.MetadataLoadContext" Version="8.0.1" />
     <PackageVersion Include="System.ServiceProcess.ServiceController" Version="8.0.1" />
+    <PackageVersion Include="Testcontainers.MsSql" Version="4.2.0" />
+    <PackageVersion Include="Testcontainers.MySql" Version="4.2.0" />
+    <PackageVersion Include="Testcontainers.PostgreSql" Version="4.2.0" />
     <PackageVersion Include="Validar.Fody" Version="1.9.0" />
     <PackageVersion Include="Yarp.ReverseProxy" Version="2.3.0" />
   </ItemGroup>

src/ProjectReferences.Persisters.Primary.props

@@ -2,6 +2,9 @@
 
   <ItemGroup Label="Persisters">
     <ProjectReference Include="..\ServiceControl.Persistence.RavenDB\ServiceControl.Persistence.RavenDB.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.Persistence.Sql\ServiceControl.Persistence.Sql.SqlServer.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.Persistence.Sql\ServiceControl.Persistence.Sql.PostgreSQL.csproj" ReferenceOutputAssembly="false" Private="false" />
+    <ProjectReference Include="..\ServiceControl.Persistence.Sql\ServiceControl.Persistence.Sql.MySQL.csproj" ReferenceOutputAssembly="false" Private="false" />
   </ItemGroup>
 
 </Project>

src/ServiceControl.Persistence.Sql.Core/.editorconfig

@@ -0,0 +1,9 @@
+[*.cs]
+
+# Justification: ServiceControl app has no synchronization context
+dotnet_diagnostic.CA2007.severity = none
+
+# Disable style rules for auto-generated EF migrations
+[**/Migrations/*.cs]
+dotnet_diagnostic.IDE0065.severity = none
+generated_code = true

src/ServiceControl.Persistence.Sql.Core/Abstractions/BasePersistence.cs

@@ -0,0 +1,43 @@
+namespace ServiceControl.Persistence.Sql.Core.Abstractions;
+
+using Microsoft.Extensions.DependencyInjection;
+using ServiceControl.Persistence;
+using ServiceControl.Persistence.MessageRedirects;
+using ServiceControl.Persistence.UnitOfWork;
+using Implementation;
+using Implementation.UnitOfWork;
+using Particular.LicensingComponent.Persistence;
+
+public abstract class BasePersistence
+{
+    protected static void RegisterDataStores(IServiceCollection services, bool maintenanceMode)
+    {
+        if (maintenanceMode)
+        {
+            return;
+        }
+
+        services.AddSingleton<MinimumRequiredStorageState>();
+        services.AddSingleton<ITrialLicenseDataProvider, TrialLicenseDataProvider>();
+        services.AddSingleton<IEndpointSettingsStore, EndpointSettingsStore>();
+        services.AddSingleton<IEventLogDataStore, EventLogDataStore>();
+        services.AddSingleton<IMessageRedirectsDataStore, MessageRedirectsDataStore>();
+        services.AddSingleton<IServiceControlSubscriptionStorage, ServiceControlSubscriptionStorage>();
+        services.AddSingleton<IQueueAddressStore, QueueAddressStore>();
+        services.AddSingleton<IMonitoringDataStore, MonitoringDataStore>();
+        services.AddSingleton<ICustomChecksDataStore, CustomChecksDataStore>();
+        services.AddSingleton<Operations.BodyStorage.IBodyStorage, BodyStorage>();
+        services.AddSingleton<IRetryHistoryDataStore, RetryHistoryDataStore>();
+        services.AddSingleton<IFailedErrorImportDataStore, FailedErrorImportDataStore>();
+        services.AddSingleton<IExternalIntegrationRequestsDataStore, ExternalIntegrationRequestsDataStore>();
+        services.AddSingleton<IFailedMessageViewIndexNotifications, FailedMessageViewIndexNotifications>();
+        services.AddSingleton<Recoverability.IArchiveMessages, ArchiveMessages>();
+        services.AddSingleton<IGroupsDataStore, GroupsDataStore>();
+        services.AddSingleton<IRetryDocumentDataStore, RetryDocumentDataStore>();
+        services.AddSingleton<IRetryBatchesDataStore, RetryBatchesDataStore>();
+        services.AddSingleton<INotificationsManager, NotificationsManager>();
+        services.AddSingleton<IIngestionUnitOfWorkFactory, IngestionUnitOfWorkFactory>();
+        services.AddSingleton<IErrorMessageDataStore, ErrorMessageDataStore>();
+        services.AddSingleton<ILicensingDataStore, LicensingDataStore>();
+    }
+}

src/ServiceControl.Persistence.Sql.Core/Abstractions/IDatabaseMigrator.cs

@@ -0,0 +1,6 @@
+namespace ServiceControl.Persistence.Sql.Core.Abstractions;
+
+public interface IDatabaseMigrator
+{
+    Task ApplyMigrations(CancellationToken cancellationToken = default);
+}

src/ServiceControl.Persistence.Sql.Core/Abstractions/SqlPersisterSettings.cs

@@ -0,0 +1,10 @@
+namespace ServiceControl.Persistence.Sql.Core.Abstractions;
+
+using ServiceControl.Persistence;
+
+public abstract class SqlPersisterSettings : PersistenceSettings
+{
+    public required string ConnectionString { get; set; }
+    public int CommandTimeout { get; set; } = 30;
+    public bool EnableSensitiveDataLogging { get; set; } = false;
+}

src/ServiceControl.Persistence.Sql.Core/DbContexts/ServiceControlDbContextBase.cs

@@ -0,0 +1,69 @@
+namespace ServiceControl.Persistence.Sql.Core.DbContexts;
+
+using Entities;
+using EntityConfigurations;
+using Microsoft.EntityFrameworkCore;
+
+public abstract class ServiceControlDbContextBase : DbContext
+{
+    protected ServiceControlDbContextBase(DbContextOptions options) : base(options)
+    {
+    }
+
+    public DbSet<TrialLicenseEntity> TrialLicenses { get; set; }
+    public DbSet<EndpointSettingsEntity> EndpointSettings { get; set; }
+    public DbSet<EventLogItemEntity> EventLogItems { get; set; }
+    public DbSet<MessageRedirectsEntity> MessageRedirects { get; set; }
+    public DbSet<SubscriptionEntity> Subscriptions { get; set; }
+    public DbSet<QueueAddressEntity> QueueAddresses { get; set; }
+    public DbSet<KnownEndpointEntity> KnownEndpoints { get; set; }
+    public DbSet<CustomCheckEntity> CustomChecks { get; set; }
+    public DbSet<MessageBodyEntity> MessageBodies { get; set; }
+    public DbSet<RetryHistoryEntity> RetryHistory { get; set; }
+    public DbSet<FailedErrorImportEntity> FailedErrorImports { get; set; }
+    public DbSet<ExternalIntegrationDispatchRequestEntity> ExternalIntegrationDispatchRequests { get; set; }
+    public DbSet<ArchiveOperationEntity> ArchiveOperations { get; set; }
+    public DbSet<FailedMessageEntity> FailedMessages { get; set; }
+    public DbSet<RetryBatchEntity> RetryBatches { get; set; }
+    public DbSet<FailedMessageRetryEntity> FailedMessageRetries { get; set; }
+    public DbSet<GroupCommentEntity> GroupComments { get; set; }
+    public DbSet<RetryBatchNowForwardingEntity> RetryBatchNowForwarding { get; set; }
+    public DbSet<NotificationsSettingsEntity> NotificationsSettings { get; set; }
+    public DbSet<LicensingMetadataEntity> LicensingMetadata { get; set; }
+    public DbSet<ThroughputEndpointEntity> Endpoints { get; set; }
+    public DbSet<DailyThroughputEntity> Throughput { get; set; }
+
+    protected override void OnModelCreating(ModelBuilder modelBuilder)
+    {
+        base.OnModelCreating(modelBuilder);
+
+        modelBuilder.ApplyConfiguration(new TrialLicenseConfiguration());
+        modelBuilder.ApplyConfiguration(new EndpointSettingsConfiguration());
+        modelBuilder.ApplyConfiguration(new EventLogItemConfiguration());
+        modelBuilder.ApplyConfiguration(new MessageRedirectsConfiguration());
+        modelBuilder.ApplyConfiguration(new SubscriptionConfiguration());
+        modelBuilder.ApplyConfiguration(new QueueAddressConfiguration());
+        modelBuilder.ApplyConfiguration(new KnownEndpointConfiguration());
+        modelBuilder.ApplyConfiguration(new CustomCheckConfiguration());
+        modelBuilder.ApplyConfiguration(new MessageBodyConfiguration());
+        modelBuilder.ApplyConfiguration(new RetryHistoryConfiguration());
+        modelBuilder.ApplyConfiguration(new FailedErrorImportConfiguration());
+        modelBuilder.ApplyConfiguration(new ExternalIntegrationDispatchRequestConfiguration());
+        modelBuilder.ApplyConfiguration(new ArchiveOperationConfiguration());
+        modelBuilder.ApplyConfiguration(new FailedMessageConfiguration());
+        modelBuilder.ApplyConfiguration(new RetryBatchConfiguration());
+        modelBuilder.ApplyConfiguration(new FailedMessageRetryConfiguration());
+        modelBuilder.ApplyConfiguration(new GroupCommentConfiguration());
+        modelBuilder.ApplyConfiguration(new RetryBatchNowForwardingConfiguration());
+        modelBuilder.ApplyConfiguration(new NotificationsSettingsConfiguration());
+        modelBuilder.ApplyConfiguration(new LicensingMetadataEntityConfiguration());
+        modelBuilder.ApplyConfiguration(new ThroughputEndpointConfiguration());
+        modelBuilder.ApplyConfiguration(new DailyThroughputConfiguration());
+
+        OnModelCreatingProvider(modelBuilder);
+    }
+
+    protected virtual void OnModelCreatingProvider(ModelBuilder modelBuilder)
+    {
+    }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/ArchiveOperationEntity.cs

@@ -0,0 +1,19 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class ArchiveOperationEntity
+{
+    public Guid Id { get; set; }
+    public string RequestId { get; set; } = null!;
+    public string GroupName { get; set; } = null!;
+    public int ArchiveType { get; set; }  // ArchiveType enum as int
+    public int ArchiveState { get; set; }  // ArchiveState enum as int
+    public int TotalNumberOfMessages { get; set; }
+    public int NumberOfMessagesArchived { get; set; }
+    public int NumberOfBatches { get; set; }
+    public int CurrentBatch { get; set; }
+    public DateTime Started { get; set; }
+    public DateTime? Last { get; set; }
+    public DateTime? CompletionTime { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/CustomCheckEntity.cs

@@ -0,0 +1,14 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+public class CustomCheckEntity
+{
+    public Guid Id { get; set; }
+    public string CustomCheckId { get; set; } = null!;
+    public string? Category { get; set; }
+    public int Status { get; set; } // 0 = Pass, 1 = Fail
+    public DateTime ReportedAt { get; set; }
+    public string? FailureReason { get; set; }
+    public string EndpointName { get; set; } = null!;
+    public Guid HostId { get; set; }
+    public string Host { get; set; } = null!;
+}

src/ServiceControl.Persistence.Sql.Core/Entities/DailyThroughputEntity.cs

@@ -0,0 +1,10 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+public class DailyThroughputEntity
+{
+    public int Id { get; set; }
+    public required string EndpointName { get; set; }
+    public required string ThroughputSource { get; set; }
+    public required DateOnly Date { get; set; }
+    public required long MessageCount { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/EndpointSettingsEntity.cs

@@ -0,0 +1,7 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+public class EndpointSettingsEntity
+{
+    public required string Name { get; set; }
+    public bool TrackInstances { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/EventLogItemEntity.cs

@@ -0,0 +1,14 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class EventLogItemEntity
+{
+    public Guid Id { get; set; }
+    public required string Description { get; set; }
+    public int Severity { get; set; }
+    public DateTime RaisedAt { get; set; }
+    public string? RelatedToJson { get; set; } // Stored as JSON array
+    public string? Category { get; set; }
+    public string? EventType { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/ExternalIntegrationDispatchRequestEntity.cs

@@ -0,0 +1,10 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class ExternalIntegrationDispatchRequestEntity
+{
+    public long Id { get; set; }
+    public string DispatchContextJson { get; set; } = null!;
+    public DateTime CreatedAt { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/FailedErrorImportEntity.cs

@@ -0,0 +1,10 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class FailedErrorImportEntity
+{
+    public Guid Id { get; set; }
+    public string MessageJson { get; set; } = null!; // FailedTransportMessage as JSON
+    public string? ExceptionInfo { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/FailedMessageEntity.cs

@@ -0,0 +1,33 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+using ServiceControl.MessageFailures;
+
+public class FailedMessageEntity
+{
+    public Guid Id { get; set; }
+    public string UniqueMessageId { get; set; } = null!;
+    public FailedMessageStatus Status { get; set; }
+
+    // JSON columns for complex nested data
+    public string ProcessingAttemptsJson { get; set; } = null!;
+    public string FailureGroupsJson { get; set; } = null!;
+    public string HeadersJson { get; set; } = null!;
+
+    // Denormalized fields from FailureGroups for efficient filtering
+    // PrimaryFailureGroupId is the first group ID from FailureGroupsJson array
+    public string? PrimaryFailureGroupId { get; set; }
+
+    // Denormalized fields from the last processing attempt for efficient querying
+    public string? MessageId { get; set; }
+    public string? MessageType { get; set; }
+    public DateTime? TimeSent { get; set; }
+    public string? SendingEndpointName { get; set; }
+    public string? ReceivingEndpointName { get; set; }
+    public string? ExceptionType { get; set; }
+    public string? ExceptionMessage { get; set; }
+    public string? QueueAddress { get; set; }
+    public int? NumberOfProcessingAttempts { get; set; }
+    public DateTime? LastProcessedAt { get; set; }
+    public string? ConversationId { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/FailedMessageRetryEntity.cs

@@ -0,0 +1,11 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class FailedMessageRetryEntity
+{
+    public Guid Id { get; set; }
+    public string FailedMessageId { get; set; } = null!;
+    public string? RetryBatchId { get; set; }
+    public int StageAttempts { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/GroupCommentEntity.cs

@@ -0,0 +1,10 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class GroupCommentEntity
+{
+    public Guid Id { get; set; }
+    public string GroupId { get; set; } = null!;
+    public string Comment { get; set; } = null!;
+}

src/ServiceControl.Persistence.Sql.Core/Entities/KnownEndpointEntity.cs

@@ -0,0 +1,11 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+public class KnownEndpointEntity
+{
+    public Guid Id { get; set; }
+    public string EndpointName { get; set; } = null!;
+    public Guid HostId { get; set; }
+    public string Host { get; set; } = null!;
+    public string HostDisplayName { get; set; } = null!;
+    public bool Monitored { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/LicensingMetadataEntity.cs

@@ -0,0 +1,8 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+public class LicensingMetadataEntity
+{
+    public int Id { get; set; }
+    public required string Key { get; set; }
+    public required string Data { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/MessageBodyEntity.cs

@@ -0,0 +1,12 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class MessageBodyEntity
+{
+    public Guid Id { get; set; }
+    public byte[] Body { get; set; } = null!;
+    public string ContentType { get; set; } = null!;
+    public int BodySize { get; set; }
+    public string? Etag { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/MessageRedirectsEntity.cs

@@ -0,0 +1,11 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class MessageRedirectsEntity
+{
+    public Guid Id { get; set; }
+    public required string ETag { get; set; }
+    public DateTime LastModified { get; set; }
+    public required string RedirectsJson { get; set; }
+}

src/ServiceControl.Persistence.Sql.Core/Entities/NotificationsSettingsEntity.cs

@@ -0,0 +1,9 @@
+namespace ServiceControl.Persistence.Sql.Core.Entities;
+
+using System;
+
+public class NotificationsSettingsEntity
+{
+    public Guid Id { get; set; }
+    public string EmailSettingsJson { get; set; } = string.Empty;
+}