diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 29f3ae023a..fa44b13aa2 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -9,6 +9,7 @@ + @@ -20,6 +21,10 @@ + + + + @@ -52,6 +57,7 @@ + @@ -92,4 +98,4 @@ - + \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/.editorconfig b/src/ServiceControl.Audit.Persistence.Sql.Core/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/AuditSqlPersisterSettings.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/AuditSqlPersisterSettings.cs new file mode 100644 index 0000000000..7893f9ecc3 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/AuditSqlPersisterSettings.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Abstractions; + +public abstract class AuditSqlPersisterSettings : PersistenceSettings +{ + protected AuditSqlPersisterSettings( + TimeSpan auditRetentionPeriod, + bool enableFullTextSearchOnBodies, + int maxBodySizeToStore) + : base(auditRetentionPeriod, enableFullTextSearchOnBodies, maxBodySizeToStore) + { + } + + public required string ConnectionString { get; set; } + public int CommandTimeout { get; set; } = 30; + public bool EnableSensitiveDataLogging { get; set; } = false; + public int MinBodySizeForCompression { get; set; } = 4096; + public bool StoreMessageBodiesOnDisk { get; set; } = true; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/BaseAuditPersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/BaseAuditPersistence.cs new file mode 100644 index 0000000000..da04f5f6a4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/BaseAuditPersistence.cs @@ -0,0 +1,30 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Abstractions; + +using Azure.Storage.Blobs; +using Implementation; +using Implementation.UnitOfWork; +using Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Auditing.BodyStorage; +using ServiceControl.Audit.Persistence.UnitOfWork; + +public abstract class BaseAuditPersistence +{ + protected static void RegisterDataStores(IServiceCollection services, AuditSqlPersisterSettings settings) + { + services.AddSingleton(); + if (!string.IsNullOrEmpty(settings.MessageBodyStoragePath)) + { + services.AddSingleton(); + } + else + { + services.AddSingleton(); + } + services.AddScoped(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(TimeProvider.System); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/IAuditDatabaseMigrator.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/IAuditDatabaseMigrator.cs new file mode 100644 index 0000000000..e987af8f5b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/IAuditDatabaseMigrator.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Abstractions; + +public interface IAuditDatabaseMigrator +{ + Task ApplyMigrations(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/MinimumRequiredStorageState.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/MinimumRequiredStorageState.cs new file mode 100644 index 0000000000..72fd05b84a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Abstractions/MinimumRequiredStorageState.cs @@ -0,0 +1,6 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Abstractions; + +public class MinimumRequiredStorageState +{ + public bool CanIngestMore { get; set; } = true; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/DbContexts/AuditDbContextBase.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/DbContexts/AuditDbContextBase.cs new file mode 100644 index 0000000000..36211fd95e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/DbContexts/AuditDbContextBase.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.DbContexts; + +using Entities; +using EntityConfigurations; +using Microsoft.EntityFrameworkCore; + +public abstract class AuditDbContextBase : DbContext +{ + protected AuditDbContextBase(DbContextOptions options) : base(options) + { + } + + public DbSet ProcessedMessages { get; set; } + public DbSet FailedAuditImports { get; set; } + public DbSet SagaSnapshots { get; set; } + public DbSet KnownEndpoints { get; set; } + public DbSet KnownEndpointsInsertOnly { get; set; } + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + optionsBuilder.EnableDetailedErrors(); + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + base.OnModelCreating(modelBuilder); + + modelBuilder.ApplyConfiguration(new ProcessedMessageConfiguration()); + modelBuilder.ApplyConfiguration(new FailedAuditImportConfiguration()); + modelBuilder.ApplyConfiguration(new SagaSnapshotConfiguration()); + modelBuilder.ApplyConfiguration(new KnownEndpointConfiguration()); + modelBuilder.ApplyConfiguration(new KnownEndpointInsertOnlyConfiguration()); + + OnModelCreatingProvider(modelBuilder); + } + + protected virtual void OnModelCreatingProvider(ModelBuilder modelBuilder) + { + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/FailedAuditImportEntity.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/FailedAuditImportEntity.cs new file mode 100644 index 0000000000..8292a48cf2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/FailedAuditImportEntity.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Entities; + +public class FailedAuditImportEntity +{ + public Guid Id { get; set; } + public string MessageJson { get; set; } = null!; + public string? ExceptionInfo { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointEntity.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointEntity.cs new file mode 100644 index 0000000000..21ef329ca7 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointEntity.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Entities; + +public class KnownEndpointEntity +{ + public Guid Id { get; set; } + public string? Name { get; set; } + + public Guid HostId { get; set; } + + public string? Host { get; set; } + + public DateTime LastSeen { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointInsertOnlyEntity.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointInsertOnlyEntity.cs new file mode 100644 index 0000000000..0f6fb46c5c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/KnownEndpointInsertOnlyEntity.cs @@ -0,0 +1,15 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Entities; + +public class KnownEndpointInsertOnlyEntity +{ + public long Id { get; set; } + public Guid KnownEndpointId { get; set; } + + public string? Name { get; set; } + + public Guid HostId { get; set; } + + public string? Host { get; set; } + + public DateTime LastSeen { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/ProcessedMessageEntity.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/ProcessedMessageEntity.cs new file mode 100644 index 0000000000..cecc8d6715 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/ProcessedMessageEntity.cs @@ -0,0 +1,35 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Entities; + +public class ProcessedMessageEntity +{ + public long Id { get; set; } + public string UniqueMessageId { get; set; } = null!; + + // JSON columns for complex nested data + public string HeadersJson { get; set; } = null!; + + // Full-text search column (combines headers JSON and message body for indexing) + public string? SearchableContent { get; set; } + + // Denormalized fields for efficient querying + public string? MessageId { get; set; } + public string? MessageType { get; set; } + public DateTime? TimeSent { get; set; } + public DateTime CreatedOn { get; set; } + public bool IsSystemMessage { get; set; } + public int Status { get; set; } + public string? ConversationId { get; set; } + + // Endpoint details (denormalized from MessageMetadata) + public string? ReceivingEndpointName { get; set; } + + // Performance metrics (stored as ticks for precision) + public long? CriticalTimeTicks { get; set; } + public long? ProcessingTimeTicks { get; set; } + public long? DeliveryTimeTicks { get; set; } + + // Body storage info + public int BodySize { get; set; } + public string? BodyUrl { get; set; } + public bool BodyNotStored { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/SagaSnapshotEntity.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/SagaSnapshotEntity.cs new file mode 100644 index 0000000000..4b158572bb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Entities/SagaSnapshotEntity.cs @@ -0,0 +1,18 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Entities; + +using ServiceControl.SagaAudit; + +public class SagaSnapshotEntity +{ + public long Id { get; set; } + public Guid SagaId { get; set; } + public string? SagaType { get; set; } + public DateTime StartTime { get; set; } + public DateTime FinishTime { get; set; } + public SagaStateChangeStatus Status { get; set; } + public string? StateAfterChange { get; set; } + public string? InitiatingMessageJson { get; set; } + public string? OutgoingMessagesJson { get; set; } + public string? Endpoint { get; set; } + public DateTime CreatedOn { get; set; } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/FailedAuditImportConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/FailedAuditImportConfiguration.cs new file mode 100644 index 0000000000..21f1cca11f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/FailedAuditImportConfiguration.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class FailedAuditImportConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("FailedAuditImports"); + builder.HasKey(e => e.Id); + builder.Property(e => e.Id).IsRequired(); + builder.Property(e => e.MessageJson).IsRequired(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointConfiguration.cs new file mode 100644 index 0000000000..de2253de2d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointConfiguration.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class KnownEndpointConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("KnownEndpoints"); + builder.HasKey(e => e.Id); + builder.Property(e => e.Id).ValueGeneratedNever(); + builder.Property(e => e.Name).IsRequired(); + builder.Property(e => e.HostId).IsRequired(); + builder.Property(e => e.Host).IsRequired(); + builder.Property(e => e.LastSeen).IsRequired(); + + builder.HasIndex(e => e.LastSeen); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs new file mode 100644 index 0000000000..e2d10ff6ab --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/KnownEndpointInsertOnlyConfiguration.cs @@ -0,0 +1,23 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class KnownEndpointInsertOnlyConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("KnownEndpointsInsertOnly"); + builder.HasKey(e => e.Id); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + builder.Property(e => e.KnownEndpointId).IsRequired(); + builder.Property(e => e.Name).IsRequired(); + builder.Property(e => e.HostId).IsRequired(); + builder.Property(e => e.Host).IsRequired(); + builder.Property(e => e.LastSeen).IsRequired(); + + builder.HasIndex(e => e.LastSeen); + builder.HasIndex(e => e.KnownEndpointId); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/ProcessedMessageConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/ProcessedMessageConfiguration.cs new file mode 100644 index 0000000000..9ef6a92220 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/ProcessedMessageConfiguration.cs @@ -0,0 +1,43 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class ProcessedMessageConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("ProcessedMessages"); + builder.HasKey(e => new { e.Id, e.CreatedOn }); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + builder.Property(e => e.CreatedOn).IsRequired(); + builder.Property(e => e.UniqueMessageId).HasMaxLength(200).IsRequired(); + + // JSON columns + builder.Property(e => e.HeadersJson).IsRequired(); + + // Full-text search column (combines header values + body text) + builder.Property(e => e.SearchableContent); + + // Denormalized query fields + builder.Property(e => e.MessageId).HasMaxLength(200); + builder.Property(e => e.MessageType).HasMaxLength(500); + builder.Property(e => e.ConversationId).HasMaxLength(200); + builder.Property(e => e.ReceivingEndpointName).HasMaxLength(500); + builder.Property(e => e.BodyUrl).HasMaxLength(500); + builder.Property(e => e.TimeSent); + builder.Property(e => e.IsSystemMessage).IsRequired(); + builder.Property(e => e.Status).IsRequired(); + builder.Property(e => e.BodySize).IsRequired(); + builder.Property(e => e.BodyNotStored).IsRequired(); + builder.Property(e => e.CriticalTimeTicks); + builder.Property(e => e.ProcessingTimeTicks); + builder.Property(e => e.DeliveryTimeTicks); + + builder.HasIndex(e => e.UniqueMessageId); + builder.HasIndex(e => e.ConversationId); + builder.HasIndex(e => e.MessageId); + builder.HasIndex(e => e.TimeSent); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/SagaSnapshotConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/SagaSnapshotConfiguration.cs new file mode 100644 index 0000000000..24f3e249b4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/EntityConfigurations/SagaSnapshotConfiguration.cs @@ -0,0 +1,27 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.EntityConfigurations; + +using Entities; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +class SagaSnapshotConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.ToTable("SagaSnapshots"); + builder.HasKey(e => new { e.Id, e.CreatedOn }); + builder.Property(e => e.Id).ValueGeneratedOnAdd(); + builder.Property(e => e.CreatedOn).IsRequired(); + builder.Property(e => e.SagaId).IsRequired(); + builder.Property(e => e.SagaType).IsRequired(); + builder.Property(e => e.StartTime).IsRequired(); + builder.Property(e => e.FinishTime); + builder.Property(e => e.Status).IsRequired(); + builder.Property(e => e.StateAfterChange).IsRequired(); + builder.Property(e => e.InitiatingMessageJson).IsRequired(); + builder.Property(e => e.OutgoingMessagesJson).IsRequired(); + builder.Property(e => e.Endpoint).IsRequired(); + + builder.HasIndex(e => e.SagaId); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/FullTextSearch/IAuditFullTextSearchProvider.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/FullTextSearch/IAuditFullTextSearchProvider.cs new file mode 100644 index 0000000000..ef303ba399 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/FullTextSearch/IAuditFullTextSearchProvider.cs @@ -0,0 +1,10 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.FullTextSearch; + +using Entities; + +public interface IAuditFullTextSearchProvider +{ + IQueryable ApplyFullTextSearch( + IQueryable query, + string searchTerms); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/AzureBlobBodyStoragePersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/AzureBlobBodyStoragePersistence.cs new file mode 100644 index 0000000000..ca75116a7f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/AzureBlobBodyStoragePersistence.cs @@ -0,0 +1,147 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using System.Buffers; +using System.IO.Compression; +using Azure.Storage; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using ServiceControl.Audit.Persistence.Sql.Core.Abstractions; + +public class AzureBlobBodyStoragePersistence : IBodyStoragePersistence +{ + const string FormatVersion = "1"; + readonly AuditSqlPersisterSettings settings; + readonly BlobContainerClient blobContainerClient; + + public AzureBlobBodyStoragePersistence(AuditSqlPersisterSettings settings) + { + this.settings = settings; + + var blobClient = new BlobServiceClient(settings.MessageBodyStorageConnectionString); + blobContainerClient = blobClient.GetBlobContainerClient("audit-bodies"); + } + + public async Task WriteBodyAsync(string bodyId, DateTime createdOn, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default) + { + var datePrefix = createdOn.ToString("yyyy-MM-dd-HH"); + var blob = blobContainerClient.GetBlobClient($"{datePrefix}/{bodyId}"); + var shouldCompress = body.Length >= settings.MinBodySizeForCompression; + + BinaryData data; + byte[]? rentedBuffer = null; + + try + { + if (shouldCompress) + { + var maxCompressedSize = BrotliEncoder.GetMaxCompressedLength(body.Length); + rentedBuffer = ArrayPool.Shared.Rent(maxCompressedSize); + + if (!BrotliEncoder.TryCompress(body.Span, rentedBuffer, out var bytesWritten, quality: 1, window: 22)) + { + // Compression failed, fall back to uncompressed + data = BinaryData.FromBytes(body); + shouldCompress = false; + } + else + { + data = BinaryData.FromBytes(new ReadOnlyMemory(rentedBuffer, 0, bytesWritten)); + } + } + else + { + data = BinaryData.FromBytes(body); + } + + var options = new BlobUploadOptions + { + TransferValidation = new UploadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.Auto + }, + Metadata = new Dictionary + { + { "FormatVersion", FormatVersion }, + { "ContentType", Uri.EscapeDataString(contentType) }, + { "BodySize", body.Length.ToString() }, + { "IsCompressed", shouldCompress.ToString() } + } + }; + + await blob.UploadAsync(data, options, cancellationToken); + } + finally + { + if (rentedBuffer != null) + { + ArrayPool.Shared.Return(rentedBuffer); + } + } + } + + public async Task ReadBodyAsync(string bodyId, DateTime createdOn, CancellationToken cancellationToken = default) + { + var datePrefix = createdOn.ToString("yyyy-MM-dd-HH"); + var blob = blobContainerClient.GetBlobClient($"{datePrefix}/{bodyId}"); + + try + { + var response = await blob.DownloadContentAsync(cancellationToken); + var properties = response.Value; + var metadata = properties.Details.Metadata; + + // Check format version + if (metadata.TryGetValue("FormatVersion", out var version) && version != FormatVersion) + { + throw new InvalidOperationException($"Unsupported blob format version: {version}"); + } + + var contentType = metadata.TryGetValue("ContentType", out var ct) ? Uri.UnescapeDataString(ct) : "application/octet-stream"; + var bodySize = metadata.TryGetValue("BodySize", out var sizeStr) && int.TryParse(sizeStr, out var size) ? size : 0; + var isCompressed = metadata.TryGetValue("IsCompressed", out var compressedStr) && bool.TryParse(compressedStr, out var compressed) && compressed; + var etag = properties.Details.ETag.ToString(); + + Stream stream; + if (isCompressed) + { + var compressedData = properties.Content.ToMemory(); + var decompressedBuffer = new byte[bodySize]; + + if (!BrotliDecoder.TryDecompress(compressedData.Span, decompressedBuffer, out var bytesWritten) || bytesWritten != bodySize) + { + throw new InvalidOperationException($"Failed to decompress body for {bodyId}"); + } + + stream = new MemoryStream(decompressedBuffer, writable: false); + } + else + { + stream = properties.Content.ToStream(); + } + + return new MessageBodyFileResult + { + Stream = stream, + ContentType = contentType, + BodySize = bodySize, + Etag = etag + }; + } + catch (Azure.RequestFailedException ex) when (ex.Status == 404) + { + return null; + } + } + + public Task DeleteBodiesForHour(DateTime hour, CancellationToken cancellationToken = default) + { + // var hourPrefix = hour.ToString("yyyy-MM-dd-HH") + "/"; + + // await foreach (var blobItem in blobContainerClient.GetBlobsAsync(BlobTraits.None, BlobStates.None, hourPrefix, cancellationToken)) + // { + // await blobContainerClient.DeleteBlobIfExistsAsync(blobItem.Name, cancellationToken: cancellationToken); + // } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/BodyStorageFetcher.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/BodyStorageFetcher.cs new file mode 100644 index 0000000000..a54b4a6f46 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/BodyStorageFetcher.cs @@ -0,0 +1,44 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Auditing.BodyStorage; +using ServiceControl.Audit.Persistence.Sql.Core.DbContexts; +using ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +class BodyStorageFetcher(IBodyStoragePersistence storagePersistence, AuditDbContextBase dbContext) : IBodyStorage +{ + public async Task Store(string bodyId, string contentType, int bodySize, Stream bodyStream, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + public async Task TryFetch(string bodyId, CancellationToken cancellationToken) + { + // Look up CreatedOn from the database to locate the correct hourly folder + var createdOn = await dbContext.ProcessedMessages + .Where(m => m.UniqueMessageId == bodyId) + .Select(m => m.CreatedOn) + .FirstOrDefaultAsync(cancellationToken); + + if (createdOn == default) + { + return new StreamResult { HasResult = false }; + } + + var result = await storagePersistence.ReadBodyAsync(bodyId, createdOn, cancellationToken); + + if (result == null) + { + return new StreamResult { HasResult = false }; + } + + return new StreamResult + { + HasResult = true, + Stream = result.Stream, + ContentType = result.ContentType, + BodySize = result.BodySize, + Etag = result.Etag + }; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFAuditDataStore.cs new file mode 100644 index 0000000000..8b805955b8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFAuditDataStore.cs @@ -0,0 +1,39 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation; + +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Auditing.MessagesView; +using ServiceControl.Audit.Infrastructure; +using ServiceControl.Audit.Monitoring; +using ServiceControl.SagaAudit; + +class EFAuditDataStore : IAuditDataStore +{ + static readonly QueryStatsInfo EmptyStats = new(string.Empty, 0); + + public Task>> QueryKnownEndpoints(CancellationToken cancellationToken) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken) + => Task.FromResult(QueryResult.Empty()); + + public Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange? timeSentRange = null, CancellationToken cancellationToken = default) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + => Task.FromResult(new QueryResult>([], EmptyStats)); + + public Task GetMessageBody(string messageId, CancellationToken cancellationToken) + => Task.FromResult(MessageBodyView.NoContent()); + + public Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken) + => Task.FromResult(new QueryResult>([], EmptyStats)); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFFailedAuditStorage.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFFailedAuditStorage.cs new file mode 100644 index 0000000000..fdc6e2f4b1 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/EFFailedAuditStorage.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation; + +using ServiceControl.Audit.Auditing; + +class EFFailedAuditStorage : IFailedAuditStorage +{ + public Task SaveFailedAuditImport(FailedAuditImport message) + => Task.CompletedTask; + + public Task ProcessFailedMessages( + Func, CancellationToken, Task> onMessage, + CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task GetFailedAuditsCount() + => Task.FromResult(0); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/FileSystemBodyStoragePersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/FileSystemBodyStoragePersistence.cs new file mode 100644 index 0000000000..f72e53a14f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/FileSystemBodyStoragePersistence.cs @@ -0,0 +1,181 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using System.IO.Compression; +using Abstractions; + +public class FileSystemBodyStoragePersistence(AuditSqlPersisterSettings settings) : IBodyStoragePersistence +{ + const int FormatVersion = 1; + + public async Task WriteBodyAsync( + string bodyId, + DateTime createdOn, + ReadOnlyMemory body, + string contentType, + CancellationToken cancellationToken = default) + { + var dateFolder = createdOn.ToString("yyyy-MM-dd-HH"); + var filePath = Path.Combine(settings.MessageBodyStoragePath, dateFolder, $"{bodyId}.body"); + + // Bodies are immutable - skip if file already exists + if (File.Exists(filePath)) + { + return; + } + + // Ensure directory exists + var directory = Path.GetDirectoryName(filePath); + if (!string.IsNullOrEmpty(directory) && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + // Write to temp file first for atomic operation + var tempFilePath = filePath + ".tmp"; + + try + { + var fileStream = new FileStream( + tempFilePath, + FileMode.Create, + FileAccess.Write, + FileShare.None, + bufferSize: 4096, + useAsync: true); + + await using (fileStream.ConfigureAwait(false)) + { + using var writer = new BinaryWriter(fileStream, System.Text.Encoding.UTF8, leaveOpen: true); + + var shouldCompress = body.Length >= settings.MinBodySizeForCompression; + + // Write header + writer.Write(FormatVersion); + writer.Write(contentType); + writer.Write(body.Length); // Original uncompressed size + writer.Write(shouldCompress); + writer.Write(Guid.NewGuid().ToString()); // Generate ETag + + // Flush the header before writing body + writer.Flush(); + + // Write body (compressed or not) + if (shouldCompress) + { + var brotliStream = new BrotliStream(fileStream, CompressionLevel.Fastest, leaveOpen: true); + await using (brotliStream.ConfigureAwait(false)) + { + await brotliStream.WriteAsync(body, cancellationToken).ConfigureAwait(false); + } + } + else + { + await fileStream.WriteAsync(body, cancellationToken).ConfigureAwait(false); + } + + await fileStream.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + // Atomic rename + File.Move(tempFilePath, filePath, overwrite: false); + } + catch + { + // Clean up temp file if it exists + if (File.Exists(tempFilePath)) + { + try + { + File.Delete(tempFilePath); + } + catch + { + // Ignore cleanup errors + } + } + throw; + } + } + + public Task ReadBodyAsync(string bodyId, DateTime createdOn, CancellationToken cancellationToken = default) + { + var dateFolder = createdOn.ToString("yyyy-MM-dd-HH"); + var filePath = Path.Combine(settings.MessageBodyStoragePath, dateFolder, $"{bodyId}.body"); + + if (!File.Exists(filePath)) + { + return Task.FromResult(null); + } + + try + { + var fileStream = new FileStream( + filePath, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + bufferSize: 4096, + useAsync: true); + + var reader = new BinaryReader(fileStream, System.Text.Encoding.UTF8, leaveOpen: true); + + // Read header + var formatVersion = reader.ReadInt32(); + if (formatVersion != FormatVersion) + { + fileStream.Dispose(); + throw new InvalidOperationException($"Unsupported body file format version: {formatVersion}"); + } + + var contentType = reader.ReadString(); + var bodySize = reader.ReadInt32(); + var isCompressed = reader.ReadBoolean(); + var etag = reader.ReadString(); + + // Create appropriate stream wrapper for body data + Stream bodyStream = fileStream; + if (isCompressed) + { + bodyStream = new BrotliStream(fileStream, CompressionMode.Decompress, leaveOpen: false); + } + + var result = new MessageBodyFileResult + { + Stream = bodyStream, + ContentType = contentType, + BodySize = bodySize, + Etag = etag + }; + + return Task.FromResult(result); + } + catch (FileNotFoundException) + { + return Task.FromResult(null); + } + catch (IOException ex) + { + throw new InvalidOperationException($"Failed to read body file for {bodyId}", ex); + } + } + + public Task DeleteBodiesForHour(DateTime hour, CancellationToken cancellationToken = default) + { + var dateFolder = hour.ToString("yyyy-MM-dd-HH"); + var directoryPath = Path.Combine(settings.MessageBodyStoragePath, dateFolder); + + try + { + if (Directory.Exists(directoryPath)) + { + Directory.Delete(directoryPath, recursive: true); + } + } + catch (DirectoryNotFoundException) + { + // Already gone + } + + return Task.CompletedTask; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs new file mode 100644 index 0000000000..9a600651be --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWork.cs @@ -0,0 +1,240 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation.UnitOfWork; + +using System.Net.Mime; +using System.Text; +using System.Text.Json; +using Abstractions; +using DbContexts; +using Entities; +using Infrastructure; +using ServiceControl.Audit.Auditing; +using ServiceControl.Audit.Monitoring; +using ServiceControl.Audit.Persistence.Infrastructure; +using ServiceControl.Audit.Persistence.Monitoring; +using ServiceControl.Audit.Persistence.UnitOfWork; +using ServiceControl.SagaAudit; + +class AuditIngestionUnitOfWork( + AuditDbContextBase dbContext, + IBodyStoragePersistence bodyPersistence, + AuditSqlPersisterSettings settings + ) + : IAuditIngestionUnitOfWork +{ + readonly List bodyStorageTasks = []; + // Large object heap starts above 85000 bytes + const int LargeObjectHeapThreshold = 85_000; + static readonly Encoding Utf8 = new UTF8Encoding(true, true); + + public async Task RecordProcessedMessage(ProcessedMessage processedMessage, ReadOnlyMemory body = default, CancellationToken cancellationToken = default) + { + var createdOn = TruncateToHour(DateTime.UtcNow); + + var entity = new ProcessedMessageEntity + { + CreatedOn = createdOn, + UniqueMessageId = processedMessage.UniqueMessageId, + HeadersJson = JsonSerializer.Serialize(processedMessage.Headers, ProcessedMessageJsonContext.Default.DictionaryStringString), + + // Denormalized fields + MessageId = GetMetadata(processedMessage.MessageMetadata, "MessageId"), + MessageType = GetMetadata(processedMessage.MessageMetadata, "MessageType"), + TimeSent = GetMetadata(processedMessage.MessageMetadata, "TimeSent"), + IsSystemMessage = GetMetadata(processedMessage.MessageMetadata, "IsSystemMessage"), + Status = (int)(GetMetadata(processedMessage.MessageMetadata, "IsRetried") ? MessageStatus.ResolvedSuccessfully : MessageStatus.Successful), + ConversationId = GetMetadata(processedMessage.MessageMetadata, "ConversationId"), + + // Endpoint details + ReceivingEndpointName = GetEndpointName(processedMessage.MessageMetadata, "ReceivingEndpoint"), + + // Performance metrics + CriticalTimeTicks = GetMetadata(processedMessage.MessageMetadata, "CriticalTime")?.Ticks, + ProcessingTimeTicks = GetMetadata(processedMessage.MessageMetadata, "ProcessingTime")?.Ticks, + DeliveryTimeTicks = GetMetadata(processedMessage.MessageMetadata, "DeliveryTime")?.Ticks, + + // Full-text search content (header values + body text for single-column FTS indexing) + SearchableContent = settings.EnableFullTextSearchOnBodies ? BuildSearchableContent(processedMessage.Headers, body) : null, + BodySize = body.Length, + BodyUrl = body.IsEmpty ? null : $"/messages/{processedMessage.Id}/body", + BodyNotStored = !settings.StoreMessageBodiesOnDisk || body.Length > settings.MaxBodySizeToStore + }; + + dbContext.ProcessedMessages.Add(entity); + + //Store body if below threshold and storage is enabled + if (settings.StoreMessageBodiesOnDisk && !body.IsEmpty && body.Length < settings.MaxBodySizeToStore) + { + var contentType = GetContentType(processedMessage.Headers, MediaTypeNames.Text.Plain); + + // Queue body storage to run in parallel, awaited in DisposeAsync + bodyStorageTasks.Add(bodyPersistence.WriteBodyAsync(processedMessage.UniqueMessageId, createdOn, body, contentType, cancellationToken)); + } + } + + public Task RecordSagaSnapshot(SagaSnapshot sagaSnapshot, CancellationToken cancellationToken = default) + { + var entity = new SagaSnapshotEntity + { + CreatedOn = TruncateToHour(DateTime.UtcNow), + SagaId = sagaSnapshot.SagaId, + SagaType = sagaSnapshot.SagaType, + StartTime = sagaSnapshot.StartTime, + FinishTime = sagaSnapshot.FinishTime, + Endpoint = sagaSnapshot.Endpoint, + Status = sagaSnapshot.Status, + InitiatingMessageJson = JsonSerializer.Serialize(sagaSnapshot.InitiatingMessage, SagaSnapshotJsonContext.Default.InitiatingMessage), + OutgoingMessagesJson = JsonSerializer.Serialize(sagaSnapshot.OutgoingMessages, SagaSnapshotJsonContext.Default.ListResultingMessage), + StateAfterChange = sagaSnapshot.StateAfterChange, + }; + + dbContext.SagaSnapshots.Add(entity); + + return Task.CompletedTask; + } + + public Task RecordKnownEndpoint(KnownEndpoint knownEndpoint, CancellationToken cancellationToken = default) + { + var entity = new KnownEndpointInsertOnlyEntity + { + KnownEndpointId = DeterministicGuid.MakeId(knownEndpoint.Name, knownEndpoint.HostId.ToString()), + Name = knownEndpoint.Name, + HostId = knownEndpoint.HostId, + Host = knownEndpoint.Host, + LastSeen = knownEndpoint.LastSeen + }; + + dbContext.KnownEndpointsInsertOnly.Add(entity); + + return Task.CompletedTask; + } + + public async ValueTask DisposeAsync() + { + try + { + // Wait for all body storage operations to complete first + await Task.WhenAll(bodyStorageTasks); + await dbContext.SaveChangesAsync(); + } + finally + { + await dbContext.DisposeAsync(); + } + } + + static string GetContentType(IReadOnlyDictionary headers, string defaultContentType) + => headers.TryGetValue(Headers.ContentType, out var contentType) ? contentType : defaultContentType; + + static T? GetMetadata(Dictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var value)) + { + if (value is T typedValue) + { + return typedValue; + } + + // Handle JSON deserialized types + if (value is JsonElement jsonElement) + { + return DeserializeJsonElement(jsonElement); + } + } + return default; + } + + static T? DeserializeJsonElement(JsonElement element) + { + try + { + return element.Deserialize(JsonSerializationOptions.Default); + } + catch + { + return default; + } + } + + static string? GetEndpointName(Dictionary metadata, string key) + { + if (metadata.TryGetValue(key, out var value)) + { + if (value is EndpointDetails endpoint) + { + return endpoint.Name; + } + + if (value is JsonElement jsonElement) + { + try + { + var endpoint2 = jsonElement.Deserialize(JsonSerializationOptions.Default); + return endpoint2?.Name; + } + catch + { + return null; + } + } + } + return null; + } + + static string BuildSearchableContent(Dictionary headers, ReadOnlyMemory body) + { + // Combine header values (not keys) with body text for FTS indexing + var headerValues = string.Join(" ", headers.Values); + + var bodyString = GetBodyAsText(headers, body); + if (string.IsNullOrWhiteSpace(bodyString)) + { + return headerValues; + } + + return headerValues + " " + bodyString; + } + + static string? GetBodyAsText(Dictionary headers, ReadOnlyMemory body) + { + if (body.IsEmpty) + { + return null; + } + + var avoidsLargeObjectHeap = body.Length < LargeObjectHeapThreshold; + var isBinary = IsBinaryContent(headers); + + if (avoidsLargeObjectHeap && !isBinary) + { + try + { + var bodyString = Utf8.GetString(body.Span); + if (!string.IsNullOrWhiteSpace(bodyString)) + { + return bodyString; + } + } + catch + { + // If it won't decode to text, don't index it + } + } + + return null; + } + + static bool IsBinaryContent(Dictionary headers) + { + if (headers.TryGetValue(Headers.ContentType, out var contentType)) + { + return contentType.Contains("octet-stream") || + contentType.Contains("application/x-") || + contentType.Contains("image/") || + contentType.Contains("audio/") || + contentType.Contains("video/"); + } + return false; + } + + static DateTime TruncateToHour(DateTime dt) => new(dt.Year, dt.Month, dt.Day, dt.Hour, 0, 0, dt.Kind); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs new file mode 100644 index 0000000000..6a9b8737f4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/AuditIngestionUnitOfWorkFactory.cs @@ -0,0 +1,25 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation.UnitOfWork; + +using Abstractions; +using DbContexts; +using Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using ServiceControl.Audit.Persistence.UnitOfWork; + +class AuditIngestionUnitOfWorkFactory( + IServiceProvider serviceProvider, + MinimumRequiredStorageState storageState, + IBodyStoragePersistence storagePersistence) + : IAuditIngestionUnitOfWorkFactory +{ + public ValueTask StartNew(int batchSize, CancellationToken cancellationToken) + { + var scope = serviceProvider.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + var settings = scope.ServiceProvider.GetRequiredService(); + var unitOfWork = new AuditIngestionUnitOfWork(dbContext, storagePersistence, settings); + return ValueTask.FromResult(unitOfWork); + } + + public bool CanIngestMore() => storageState.CanIngestMore; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/ProcessedMessageJsonContext.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/ProcessedMessageJsonContext.cs new file mode 100644 index 0000000000..e790b614d6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/ProcessedMessageJsonContext.cs @@ -0,0 +1,11 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation.UnitOfWork; + +using System.Text.Json.Serialization; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = false)] +[JsonSerializable(typeof(Dictionary))] +partial class ProcessedMessageJsonContext : JsonSerializerContext +{ +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/SagaSnapshotJsonContext.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/SagaSnapshotJsonContext.cs new file mode 100644 index 0000000000..6dbac3c2c2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Implementation/UnitOfWork/SagaSnapshotJsonContext.cs @@ -0,0 +1,13 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Implementation.UnitOfWork; + +using System.Text.Json.Serialization; +using ServiceControl.SagaAudit; + +[JsonSourceGenerationOptions( + PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase, + WriteIndented = false)] +[JsonSerializable(typeof(InitiatingMessage))] +[JsonSerializable(typeof(List))] +partial class SagaSnapshotJsonContext : JsonSerializerContext +{ +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IBodyStoragePersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IBodyStoragePersistence.cs new file mode 100644 index 0000000000..f0832e428e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IBodyStoragePersistence.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +public interface IBodyStoragePersistence +{ + Task WriteBodyAsync(string bodyId, DateTime createdOn, ReadOnlyMemory body, string contentType, CancellationToken cancellationToken = default); + Task ReadBodyAsync(string bodyId, DateTime createdOn, CancellationToken cancellationToken = default); + Task DeleteBodiesForHour(DateTime hour, CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IPartitionManager.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IPartitionManager.cs new file mode 100644 index 0000000000..6d69c3994e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/IPartitionManager.cs @@ -0,0 +1,21 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using DbContexts; + +public interface IPartitionManager +{ + /// + /// Creates hourly partitions from through + . + /// + Task EnsurePartitionsExist(AuditDbContextBase dbContext, DateTime currentHour, int hoursAhead, CancellationToken ct); + + /// + /// Drops the partition for the specified hour for both ProcessedMessages and SagaSnapshots. + /// + Task DropPartition(AuditDbContextBase dbContext, DateTime partitionHour, CancellationToken ct); + + /// + /// Returns hour-precision timestamps of all partitions older than . + /// + Task> GetExpiredPartitions(AuditDbContextBase dbContext, DateTime cutoff, CancellationToken ct); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/InsertOnlyTableReconciler.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/InsertOnlyTableReconciler.cs new file mode 100644 index 0000000000..f226a513d3 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/InsertOnlyTableReconciler.cs @@ -0,0 +1,67 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using DbContexts; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +public abstract class InsertOnlyTableReconciler( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory, + string serviceName) : BackgroundService + where TInsertOnly : class + where TTarget : class +{ + protected const int BatchSize = 1000; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("Starting {ServiceName}", serviceName); + + try + { + await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken); + + using PeriodicTimer timer = new(TimeSpan.FromSeconds(30), timeProvider); + + do + { + try + { + await Reconcile(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "Error during {ServiceName} reconciliation", serviceName); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + logger.LogInformation("Stopping {ServiceName}", serviceName); + } + } + + async Task Reconcile(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + using var scope = serviceScopeFactory.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await using var transaction = await dbContext.Database.BeginTransactionAsync(stoppingToken); + var rowsAffected = await ReconcileBatch(dbContext, stoppingToken); + await transaction.CommitAsync(stoppingToken); + + if (rowsAffected < BatchSize) + { + break; + } + + await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken); + } + } + + protected abstract Task ReconcileBatch(AuditDbContextBase dbContext, CancellationToken stoppingToken); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/JsonSerializationOptions.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/JsonSerializationOptions.cs new file mode 100644 index 0000000000..5e6269310f --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/JsonSerializationOptions.cs @@ -0,0 +1,12 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using System.Text.Json; + +static class JsonSerializationOptions +{ + public static readonly JsonSerializerOptions Default = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false + }; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/MessageBodyFileResult.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/MessageBodyFileResult.cs new file mode 100644 index 0000000000..65cf7b6597 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/MessageBodyFileResult.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +public class MessageBodyFileResult +{ + public Stream Stream { get; set; } = null!; + public string ContentType { get; set; } = null!; + public int BodySize { get; set; } + public string Etag { get; set; } = null!; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionCleaner.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionCleaner.cs new file mode 100644 index 0000000000..56a5d5b2ab --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionCleaner.cs @@ -0,0 +1,110 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using System.Data.Common; +using System.Diagnostics; +using Abstractions; +using DbContexts; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +public abstract class RetentionCleaner( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory, + AuditSqlPersisterSettings settings, + IBodyStoragePersistence bodyPersistence, + IPartitionManager partitionManager, + RetentionMetrics metrics) : BackgroundService +{ + const int HoursAhead = 6; + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + logger.LogInformation("Starting {ServiceName}", nameof(RetentionCleaner)); + + try + { + await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); + + using PeriodicTimer timer = new(TimeSpan.FromHours(1), timeProvider); + + do + { + try + { + await Clean(stoppingToken); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogError(ex, "Failed to run retention cleaner"); + } + } while (await timer.WaitForNextTickAsync(stoppingToken)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + logger.LogInformation("Stopping {ServiceName}", nameof(RetentionCleaner)); + } + } + + async Task Clean(CancellationToken stoppingToken) + { + // Use a dedicated connection for the distributed lock so it is not affected + // by connection drops or resets on the main DbContext during cleanup operations + await using var lockConnection = CreateConnection(); + await lockConnection.OpenAsync(stoppingToken); + + using var scope = serviceScopeFactory.CreateScope(); + var dbContext = scope.ServiceProvider.GetRequiredService(); + + var stopwatch = Stopwatch.StartNew(); + // Round up to whole hours since partitions are hourly + var retentionPeriod = TimeSpan.FromHours(Math.Ceiling(settings.AuditRetentionPeriod.TotalHours)); + var cutoff = timeProvider.GetUtcNow().UtcDateTime - retentionPeriod; + var now = timeProvider.GetUtcNow().UtcDateTime; + + using var cycleMetrics = metrics.BeginCleanupCycle(); + + if (!await TryAcquireLock(lockConnection, stoppingToken)) + { + logger.LogDebug("Another instance is running retention cleanup, skipping this cycle"); + metrics.RecordLockSkipped(); + return; + } + + try + { + // Ensure partitions exist for upcoming hours + await partitionManager.EnsurePartitionsExist(dbContext, now, HoursAhead, stoppingToken); + + // Find and drop expired partitions + var expiredPartitions = await partitionManager.GetExpiredPartitions(dbContext, cutoff, stoppingToken); + + foreach (var hour in expiredPartitions) + { + // Delete body storage for this hour first + await bodyPersistence.DeleteBodiesForHour(hour, stoppingToken); + + // Drop the database partition + await partitionManager.DropPartition(dbContext, hour, stoppingToken); + + metrics.RecordPartitionDropped(); + + logger.LogInformation("Dropped partition for {Hour}", hour.ToString("yyyy-MM-dd HH:00")); + } + + cycleMetrics.Complete(); + + logger.LogInformation("Retention cleanup dropped {Partitions} partition(s) in {Elapsed}", + expiredPartitions.Count, stopwatch.Elapsed.ToString(@"hh\:mm\:ss")); + } + finally + { + await ReleaseLock(lockConnection, stoppingToken); + } + } + + protected abstract DbConnection CreateConnection(); + protected abstract Task TryAcquireLock(DbConnection lockConnection, CancellationToken stoppingToken); + protected abstract Task ReleaseLock(DbConnection lockConnection, CancellationToken stoppingToken); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionMetrics.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionMetrics.cs new file mode 100644 index 0000000000..01e0e74f9c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/RetentionMetrics.cs @@ -0,0 +1,78 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +using System.Diagnostics; +using System.Diagnostics.Metrics; + +public class RetentionMetrics +{ + public const string MeterName = "Particular.ServiceControl.Audit"; + + public static readonly string CleanupDurationInstrumentName = $"{InstrumentPrefix}.cleanup_duration"; + public static readonly string PartitionsDroppedInstrumentName = $"{InstrumentPrefix}.partitions_dropped_total"; + + public RetentionMetrics(IMeterFactory meterFactory) + { + var meter = meterFactory.Create(MeterName, MeterVersion); + + cleanupDuration = meter.CreateHistogram(CleanupDurationInstrumentName, unit: "s", description: "Retention cleanup cycle duration"); + partitionsDropped = meter.CreateCounter(PartitionsDroppedInstrumentName, description: "Total partitions dropped by retention cleanup"); + consecutiveFailureGauge = meter.CreateObservableGauge($"{InstrumentPrefix}.consecutive_failures_total", () => consecutiveFailures, description: "Consecutive retention cleanup failures"); + lockSkippedCounter = meter.CreateCounter($"{InstrumentPrefix}.lock_skipped_total", description: "Number of times cleanup was skipped due to another instance holding the lock"); + } + + public CleanupCycleMetrics BeginCleanupCycle() => new(cleanupDuration, RecordCycleOutcome); + + public void RecordPartitionDropped() => partitionsDropped.Add(1); + + public void RecordLockSkipped() => lockSkippedCounter.Add(1); + + void RecordCycleOutcome(bool success) + { + if (success) + { + consecutiveFailures = 0; + } + else + { + consecutiveFailures++; + } + } + + long consecutiveFailures; + + readonly Histogram cleanupDuration; + readonly Counter partitionsDropped; +#pragma warning disable IDE0052 + readonly ObservableGauge consecutiveFailureGauge; +#pragma warning restore IDE0052 + readonly Counter lockSkippedCounter; + + const string MeterVersion = "0.1.0"; + const string InstrumentPrefix = "sc.audit.retention"; +} + +public class CleanupCycleMetrics : IDisposable +{ + readonly Histogram cleanupDuration; + readonly Action recordOutcome; + readonly Stopwatch stopwatch = Stopwatch.StartNew(); + + bool completed; + + internal CleanupCycleMetrics(Histogram cleanupDuration, Action recordOutcome) + { + this.cleanupDuration = cleanupDuration; + this.recordOutcome = recordOutcome; + } + + public void Complete() => completed = true; + + public void Dispose() + { + var result = completed ? "success" : "failed"; + var tags = new TagList { { "result", result } }; + + cleanupDuration.Record(stopwatch.Elapsed.TotalSeconds, tags); + recordOutcome(completed); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/SequentialGuidGenerator.cs b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/SequentialGuidGenerator.cs new file mode 100644 index 0000000000..0bfbccfd4a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/Infrastructure/SequentialGuidGenerator.cs @@ -0,0 +1,51 @@ +namespace ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +/// +/// Generates sequential GUIDs for database primary keys to minimize page fragmentation +/// and improve insert performance while maintaining security benefits of GUIDs. +/// +/// +/// This implementation creates time-ordered GUIDs similar to .NET 9's Guid.CreateVersion7() +/// but compatible with .NET 8. The GUIDs are ordered by timestamp to reduce B-tree page splits +/// in clustered indexes, which significantly improves insert performance compared to random GUIDs. +/// +/// Benefits: +/// - Database agnostic (works with SQL Server, PostgreSQL, MySQL, SQLite) +/// - Sequential ordering reduces page fragmentation +/// - Better insert performance than random GUIDs +/// - Can easily migrate to Guid.CreateVersion7() when upgrading to .NET 9+ +/// - No external dependencies +/// +/// Security: +/// - Still cryptographically secure (uses Guid.NewGuid() as base) +/// - Not guessable (unlike sequential integers) +/// - Safe to expose in APIs +/// +public static class SequentialGuidGenerator +{ + /// + /// Generate a sequential GUID with timestamp-based ordering for optimal database performance. + /// + /// A new GUID with sequential characteristics. + public static Guid NewSequentialGuid() + { + var guidBytes = Guid.NewGuid().ToByteArray(); + var now = DateTime.UtcNow; + + // Get timestamp in milliseconds since Unix epoch (similar to Version 7 GUIDs) + var timestamp = (long)(now - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalMilliseconds; + var timestampBytes = BitConverter.GetBytes(timestamp); + + // Reverse if little-endian to get big-endian byte order for proper sorting + if (BitConverter.IsLittleEndian) + { + Array.Reverse(timestampBytes); + } + + // Replace last 6 bytes with timestamp for sequential ordering + // This placement works well with SQL Server's GUID comparison semantics + Array.Copy(timestampBytes, 2, guidBytes, 10, 6); + + return new Guid(guidBytes); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.Core/ServiceControl.Audit.Persistence.Sql.Core.csproj b/src/ServiceControl.Audit.Persistence.Sql.Core/ServiceControl.Audit.Persistence.Sql.Core.csproj new file mode 100644 index 0000000000..573a686553 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.Core/ServiceControl.Audit.Persistence.Sql.Core.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/.editorconfig b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/KnownEndpointsReconciler.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/KnownEndpointsReconciler.cs new file mode 100644 index 0000000000..542bbd826c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/KnownEndpointsReconciler.cs @@ -0,0 +1,46 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Infrastructure; + +using Core.DbContexts; +using Core.Entities; +using Core.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +class KnownEndpointsReconciler( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory) + : InsertOnlyTableReconciler( + logger, timeProvider, serviceScopeFactory, nameof(KnownEndpointsReconciler)) +{ + protected override async Task ReconcileBatch(AuditDbContextBase dbContext, CancellationToken stoppingToken) + { + var sql = @" + WITH lock_check AS ( + SELECT pg_try_advisory_xact_lock(hashtext('known_endpoints_sync')) AS acquired + ), + deleted AS ( + DELETE FROM known_endpoints_insert_only + WHERE (SELECT acquired FROM lock_check) + AND ctid IN ( + SELECT ctid FROM known_endpoints_insert_only LIMIT @batchSize + ) + RETURNING known_endpoint_id, name, host_id, host, last_seen + ), + aggregated AS ( + SELECT DISTINCT ON (known_endpoint_id) known_endpoint_id, name, host_id, host, last_seen + FROM deleted + ORDER BY known_endpoint_id, last_seen DESC + ) + INSERT INTO known_endpoints (id, name, host_id, host, last_seen) + SELECT known_endpoint_id, name, host_id, host, last_seen + FROM aggregated + ON CONFLICT (id) DO UPDATE SET + last_seen = GREATEST(known_endpoints.last_seen, EXCLUDED.last_seen); + "; + + var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [new Npgsql.NpgsqlParameter("@batchSize", BatchSize)], stoppingToken); + return rowsAffected; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/PostgreSqlPartitionManager.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/PostgreSqlPartitionManager.cs new file mode 100644 index 0000000000..f9107de18b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/PostgreSqlPartitionManager.cs @@ -0,0 +1,89 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.Sql.Core.DbContexts; +using ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +// Partition/table names cannot be parameterized in SQL; all values come from internal constants and date formatting +#pragma warning disable EF1002, EF1003 +public class PostgreSqlPartitionManager : IPartitionManager +{ + static readonly (string ParentTable, string Prefix)[] PartitionedTables = + [ + ("processed_messages", "processed_messages"), + ("saga_snapshots", "saga_snapshots") + ]; + + public async Task EnsurePartitionsExist(AuditDbContextBase dbContext, DateTime currentHour, int hoursAhead, CancellationToken ct) + { + var truncatedHour = TruncateToHour(currentHour); + var targetHour = truncatedHour.AddHours(hoursAhead); + + for (var hour = truncatedHour; hour <= targetHour; hour = hour.AddHours(1)) + { + var nextHour = hour.AddHours(1); + var hourSuffix = hour.ToString("yyyyMMddHH"); + var hourStr = hour.ToString("yyyy-MM-dd HH:00:00"); + var nextHourStr = nextHour.ToString("yyyy-MM-dd HH:00:00"); + + foreach (var (parentTable, prefix) in PartitionedTables) + { + var partitionName = prefix + "_" + hourSuffix; + + await dbContext.Database.ExecuteSqlRawAsync( + "CREATE TABLE IF NOT EXISTS " + partitionName + + " PARTITION OF " + parentTable + + " FOR VALUES FROM ('" + hourStr + "') TO ('" + nextHourStr + "')", ct); + } + } + } + + public async Task DropPartition(AuditDbContextBase dbContext, DateTime partitionHour, CancellationToken ct) + { + var hourSuffix = TruncateToHour(partitionHour).ToString("yyyyMMddHH"); + + foreach (var (parentTable, prefix) in PartitionedTables) + { + var partitionName = prefix + "_" + hourSuffix; + + await dbContext.Database.ExecuteSqlRawAsync( + "ALTER TABLE " + parentTable + " DETACH PARTITION " + partitionName, ct); + + await dbContext.Database.ExecuteSqlRawAsync( + "DROP TABLE " + partitionName, ct); + } + } + + public async Task> GetExpiredPartitions(AuditDbContextBase dbContext, DateTime cutoff, CancellationToken ct) + { + var truncatedCutoff = TruncateToHour(cutoff); + + var partitionNames = await dbContext.Database + .SqlQueryRaw( + "SELECT c.relname AS Value " + + "FROM pg_class c " + + "INNER JOIN pg_inherits i ON c.oid = i.inhrelid " + + "INNER JOIN pg_class parent ON i.inhparent = parent.oid " + + "WHERE parent.relname = 'processed_messages' " + + "AND c.relkind = 'r' " + + "ORDER BY c.relname") + .ToListAsync(ct); + + var result = new List(); + + foreach (var name in partitionNames) + { + // Parse hour from partition name: processed_messages_yyyyMMddHH + var datePart = name.Replace("processed_messages_", ""); + if (DateTime.TryParseExact(datePart, "yyyyMMddHH", null, System.Globalization.DateTimeStyles.None, out var hour) + && hour < truncatedCutoff) + { + result.Add(hour); + } + } + + return result; + } + + static DateTime TruncateToHour(DateTime dt) => new(dt.Year, dt.Month, dt.Day, dt.Hour, 0, 0, dt.Kind); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/RetentionCleaner.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/RetentionCleaner.cs new file mode 100644 index 0000000000..1689afe28a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Infrastructure/RetentionCleaner.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Infrastructure; + +using System.Data.Common; +using Core.Abstractions; +using Core.Infrastructure; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Npgsql; + +class RetentionCleaner( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory, + AuditSqlPersisterSettings settings, + IBodyStoragePersistence bodyPersistence, + IPartitionManager partitionManager, + RetentionMetrics metrics) + : Core.Infrastructure.RetentionCleaner(logger, timeProvider, serviceScopeFactory, settings, bodyPersistence, partitionManager, metrics) +{ + readonly string connectionString = settings.ConnectionString; + + protected override DbConnection CreateConnection() => new NpgsqlConnection(connectionString); + + protected override async Task TryAcquireLock(DbConnection lockConnection, CancellationToken stoppingToken) + { + await using var cmd = lockConnection.CreateCommand(); + cmd.CommandText = "SELECT pg_try_advisory_lock(hashtext('retention_cleaner'))"; + + var result = await cmd.ExecuteScalarAsync(stoppingToken); + return result is true; + } + + protected override async Task ReleaseLock(DbConnection lockConnection, CancellationToken stoppingToken) + { + await using var cmd = lockConnection.CreateCommand(); + cmd.CommandText = "SELECT pg_advisory_unlock(hashtext('retention_cleaner'))"; + + await cmd.ExecuteNonQueryAsync(stoppingToken); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.Designer.cs new file mode 100644 index 0000000000..d0457f3931 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.Designer.cs @@ -0,0 +1,281 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + [Migration("20260214031455_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_json"); + + b.HasKey("Id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("KnownEndpointId") + .HasColumnType("uuid") + .HasColumnName("known_endpoint_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints_insert_only", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("BodyNotStored") + .HasColumnType("boolean") + .HasColumnName("body_not_stored"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("body_url"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("message_type"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SearchableContent") + .HasColumnType("text") + .HasColumnName("searchable_content"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("unique_message_id"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("processed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.cs new file mode 100644 index 0000000000..3780d85378 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031455_InitialCreate.cs @@ -0,0 +1,171 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "failed_audit_imports", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + message_json = table.Column(type: "text", nullable: false), + exception_info = table.Column(type: "text", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_failed_audit_imports", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "known_endpoints", + columns: table => new + { + id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + host_id = table.Column(type: "uuid", nullable: false), + host = table.Column(type: "text", nullable: false), + last_seen = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_known_endpoints", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "known_endpoints_insert_only", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + known_endpoint_id = table.Column(type: "uuid", nullable: false), + name = table.Column(type: "text", nullable: false), + host_id = table.Column(type: "uuid", nullable: false), + host = table.Column(type: "text", nullable: false), + last_seen = table.Column(type: "timestamp with time zone", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_known_endpoints_insert_only", x => x.id); + }); + + migrationBuilder.CreateTable( + name: "processed_messages", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + created_on = table.Column(type: "timestamp with time zone", nullable: false), + unique_message_id = table.Column(type: "character varying(200)", maxLength: 200, nullable: false), + headers_json = table.Column(type: "text", nullable: false), + searchable_content = table.Column(type: "text", nullable: true), + message_id = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + message_type = table.Column(type: "character varying(500)", maxLength: 500, nullable: true), + time_sent = table.Column(type: "timestamp with time zone", nullable: true), + is_system_message = table.Column(type: "boolean", nullable: false), + status = table.Column(type: "integer", nullable: false), + conversation_id = table.Column(type: "character varying(200)", maxLength: 200, nullable: true), + receiving_endpoint_name = table.Column(type: "character varying(500)", maxLength: 500, nullable: true), + critical_time_ticks = table.Column(type: "bigint", nullable: true), + processing_time_ticks = table.Column(type: "bigint", nullable: true), + delivery_time_ticks = table.Column(type: "bigint", nullable: true), + body_size = table.Column(type: "integer", nullable: false), + body_url = table.Column(type: "character varying(500)", maxLength: 500, nullable: true), + body_not_stored = table.Column(type: "boolean", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_processed_messages", x => new { x.id, x.created_on }); + }); + + migrationBuilder.CreateTable( + name: "saga_snapshots", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false) + .Annotation("Npgsql:ValueGenerationStrategy", NpgsqlValueGenerationStrategy.IdentityByDefaultColumn), + created_on = table.Column(type: "timestamp with time zone", nullable: false), + saga_id = table.Column(type: "uuid", nullable: false), + saga_type = table.Column(type: "text", nullable: false), + start_time = table.Column(type: "timestamp with time zone", nullable: false), + finish_time = table.Column(type: "timestamp with time zone", nullable: false), + status = table.Column(type: "integer", nullable: false), + state_after_change = table.Column(type: "text", nullable: false), + initiating_message_json = table.Column(type: "text", nullable: false), + outgoing_messages_json = table.Column(type: "text", nullable: false), + endpoint = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_saga_snapshots", x => new { x.id, x.created_on }); + }); + + migrationBuilder.CreateIndex( + name: "IX_known_endpoints_last_seen", + table: "known_endpoints", + column: "last_seen"); + + migrationBuilder.CreateIndex( + name: "IX_known_endpoints_insert_only_known_endpoint_id", + table: "known_endpoints_insert_only", + column: "known_endpoint_id"); + + migrationBuilder.CreateIndex( + name: "IX_known_endpoints_insert_only_last_seen", + table: "known_endpoints_insert_only", + column: "last_seen"); + + migrationBuilder.CreateIndex( + name: "IX_processed_messages_conversation_id", + table: "processed_messages", + column: "conversation_id"); + + migrationBuilder.CreateIndex( + name: "IX_processed_messages_message_id", + table: "processed_messages", + column: "message_id"); + + migrationBuilder.CreateIndex( + name: "IX_processed_messages_time_sent", + table: "processed_messages", + column: "time_sent"); + + migrationBuilder.CreateIndex( + name: "IX_processed_messages_unique_message_id", + table: "processed_messages", + column: "unique_message_id"); + + migrationBuilder.CreateIndex( + name: "IX_saga_snapshots_saga_id", + table: "saga_snapshots", + column: "saga_id"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "failed_audit_imports"); + + migrationBuilder.DropTable( + name: "known_endpoints"); + + migrationBuilder.DropTable( + name: "known_endpoints_insert_only"); + + migrationBuilder.DropTable( + name: "processed_messages"); + + migrationBuilder.DropTable( + name: "saga_snapshots"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.Designer.cs new file mode 100644 index 0000000000..cd7c4bcb6a --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.Designer.cs @@ -0,0 +1,281 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + [Migration("20260214031511_AddPartitioning")] + partial class AddPartitioning + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_json"); + + b.HasKey("Id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("KnownEndpointId") + .HasColumnType("uuid") + .HasColumnName("known_endpoint_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints_insert_only", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("BodyNotStored") + .HasColumnType("boolean") + .HasColumnName("body_not_stored"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("body_url"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("message_type"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SearchableContent") + .HasColumnType("text") + .HasColumnName("searchable_content"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("unique_message_id"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("processed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.cs new file mode 100644 index 0000000000..d30c186b5b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031511_AddPartitioning.cs @@ -0,0 +1,40 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + /// + public partial class AddPartitioning : Migration + { + static readonly string[] Tables = ["processed_messages", "saga_snapshots"]; + + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + foreach (var table in Tables) + { + migrationBuilder.Sql($""" + CREATE TABLE {table}_tmp (LIKE {table} INCLUDING ALL); + DROP TABLE {table}; + CREATE TABLE {table} (LIKE {table}_tmp INCLUDING ALL) PARTITION BY RANGE (created_on); + DROP TABLE {table}_tmp; + """); + } + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + foreach (var table in Tables) + { + migrationBuilder.Sql($""" + CREATE TABLE {table}_tmp (LIKE {table} INCLUDING ALL); + DROP TABLE {table}; + CREATE TABLE {table} (LIKE {table}_tmp INCLUDING ALL); + DROP TABLE {table}_tmp; + """); + } + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..6f6a3a343d --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.Designer.cs @@ -0,0 +1,281 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + [Migration("20260214031534_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_json"); + + b.HasKey("Id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("KnownEndpointId") + .HasColumnType("uuid") + .HasColumnName("known_endpoint_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints_insert_only", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("BodyNotStored") + .HasColumnType("boolean") + .HasColumnName("body_not_stored"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("body_url"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("message_type"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SearchableContent") + .HasColumnType("text") + .HasColumnName("searchable_content"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("unique_message_id"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("processed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.cs new file mode 100644 index 0000000000..ea347d2ba5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/20260214031534_AddFullTextSearch.cs @@ -0,0 +1,28 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + CREATE INDEX ix_processed_messages_searchable_content + ON processed_messages + USING GIN (to_tsvector('simple', searchable_content)); + """); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + DROP INDEX IF EXISTS ix_processed_messages_searchable_content; + """); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs new file mode 100644 index 0000000000..08cbb30210 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/Migrations/PostgreSqlAuditDbContextModelSnapshot.cs @@ -0,0 +1,278 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL.Migrations +{ + [DbContext(typeof(PostgreSqlAuditDbContext))] + partial class PostgreSqlAuditDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("ExceptionInfo") + .HasColumnType("text") + .HasColumnName("exception_info"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("message_json"); + + b.HasKey("Id"); + + b.ToTable("failed_audit_imports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uuid") + .HasColumnName("id"); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("text") + .HasColumnName("host"); + + b.Property("HostId") + .HasColumnType("uuid") + .HasColumnName("host_id"); + + b.Property("KnownEndpointId") + .HasColumnType("uuid") + .HasColumnName("known_endpoint_id"); + + b.Property("LastSeen") + .HasColumnType("timestamp with time zone") + .HasColumnName("last_seen"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text") + .HasColumnName("name"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("known_endpoints_insert_only", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("BodyNotStored") + .HasColumnType("boolean") + .HasColumnName("body_not_stored"); + + b.Property("BodySize") + .HasColumnType("integer") + .HasColumnName("body_size"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("body_url"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("conversation_id"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint") + .HasColumnName("critical_time_ticks"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint") + .HasColumnName("delivery_time_ticks"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("headers_json"); + + b.Property("IsSystemMessage") + .HasColumnType("boolean") + .HasColumnName("is_system_message"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("message_id"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("message_type"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint") + .HasColumnName("processing_time_ticks"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("character varying(500)") + .HasColumnName("receiving_endpoint_name"); + + b.Property("SearchableContent") + .HasColumnType("text") + .HasColumnName("searchable_content"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.Property("TimeSent") + .HasColumnType("timestamp with time zone") + .HasColumnName("time_sent"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("character varying(200)") + .HasColumnName("unique_message_id"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("processed_messages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint") + .HasColumnName("id"); + + NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("timestamp with time zone") + .HasColumnName("created_on"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("text") + .HasColumnName("endpoint"); + + b.Property("FinishTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("finish_time"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("initiating_message_json"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("text") + .HasColumnName("outgoing_messages_json"); + + b.Property("SagaId") + .HasColumnType("uuid") + .HasColumnName("saga_id"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("text") + .HasColumnName("saga_type"); + + b.Property("StartTime") + .HasColumnType("timestamp with time zone") + .HasColumnName("start_time"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("text") + .HasColumnName("state_after_change"); + + b.Property("Status") + .HasColumnType("integer") + .HasColumnName("status"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("saga_snapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDatabaseMigrator.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDatabaseMigrator.cs new file mode 100644 index 0000000000..9fceb0b137 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDatabaseMigrator.cs @@ -0,0 +1,29 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Core.Abstractions; +using Core.DbContexts; +using Core.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +class PostgreSqlAuditDatabaseMigrator( + AuditDbContextBase dbContext, + IPartitionManager partitionManager, + TimeProvider timeProvider, + ILogger logger) + : IAuditDatabaseMigrator +{ + public async Task ApplyMigrations(CancellationToken cancellationToken = default) + { + logger.LogInformation("Starting PostgreSQL database migration for Audit"); + + await dbContext.Database.MigrateAsync(cancellationToken).ConfigureAwait(false); + + // Ensure partitions exist before ingestion starts. + // This handles gaps from downtime — creates partitions for now + 6 hours ahead. + var now = timeProvider.GetUtcNow().UtcDateTime; + await partitionManager.EnsurePartitionsExist(dbContext, now, hoursAhead: 6, cancellationToken); + + logger.LogInformation("PostgreSQL database migration completed for Audit"); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContext.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContext.cs new file mode 100644 index 0000000000..c2a2acc9ce --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContext.cs @@ -0,0 +1,82 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Core.DbContexts; +using Core.Entities; +using Microsoft.EntityFrameworkCore; +using NpgsqlTypes; + +public class PostgreSqlAuditDbContext : AuditDbContextBase +{ + public PostgreSqlAuditDbContext(DbContextOptions options) : base(options) + { + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + // Call base first to apply entity configurations + base.OnModelCreating(modelBuilder); + + // Apply snake_case naming convention for PostgreSQL + foreach (var entity in modelBuilder.Model.GetEntityTypes()) + { + // Convert table names to snake_case + var tableName = entity.GetTableName(); + if (tableName != null) + { + entity.SetTableName(ToSnakeCase(tableName)); + } + + // Convert column names to snake_case + foreach (var property in entity.GetProperties()) + { + property.SetColumnName(ToSnakeCase(property.Name)); + } + + // Skip index name conversion - EF generates unique names and snake_case + // conversion can cause collisions due to truncation + + // Convert foreign key names to snake_case + foreach (var key in entity.GetForeignKeys()) + { + var constraintName = key.GetConstraintName(); + if (constraintName != null) + { + key.SetConstraintName(ToSnakeCase(constraintName)); + } + } + } + } + + protected override void OnModelCreatingProvider(ModelBuilder modelBuilder) + { + // FTS index will be created via raw SQL in migration since EF Core + // doesn't directly support functional GIN indexes on expressions + } + + static string ToSnakeCase(string name) + { + if (string.IsNullOrEmpty(name)) + { + return name; + } + + var result = new System.Text.StringBuilder(); + for (var i = 0; i < name.Length; i++) + { + var c = name[i]; + if (char.IsUpper(c)) + { + if (i > 0) + { + result.Append('_'); + } + result.Append(char.ToLowerInvariant(c)); + } + else + { + result.Append(c); + } + } + return result.ToString(); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContextFactory.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContextFactory.cs new file mode 100644 index 0000000000..5cf84f1516 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditDbContextFactory.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +/// +/// Design-time factory for EF Core migrations tooling. +/// +public class PostgreSqlAuditDbContextFactory : IDesignTimeDbContextFactory +{ + public PostgreSqlAuditDbContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + + // Use a default connection string for design-time operations + // This is only used when running EF Core migrations tooling + var connectionString = Environment.GetEnvironmentVariable("SERVICECONTROL_AUDIT_DATABASE_CONNECTIONSTRING") + ?? "Host=localhost;Database=servicecontrol_audit;Username=postgres;Password=postgres"; + + optionsBuilder.UseNpgsql(connectionString); + + return new PostgreSqlAuditDbContext(optionsBuilder.Options); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditFullTextSearchProvider.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditFullTextSearchProvider.cs new file mode 100644 index 0000000000..a5b45c0efb --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditFullTextSearchProvider.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Core.Entities; +using Core.FullTextSearch; +using Microsoft.EntityFrameworkCore; +using NpgsqlTypes; + +class PostgreSqlAuditFullTextSearchProvider : IAuditFullTextSearchProvider +{ + public IQueryable ApplyFullTextSearch( + IQueryable query, + string searchTerms) + { + // Use 'simple' configuration for exact matching (no stemming or stop words) + // The GIN index on to_tsvector('simple', searchable_content) will be used + return query.Where(pm => + EF.Functions.ToTsVector("simple", pm.SearchableContent!) + .Matches(EF.Functions.PlainToTsQuery("simple", searchTerms))); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistence.cs new file mode 100644 index 0000000000..fd7229c606 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistence.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Core.Abstractions; +using Core.DbContexts; +using Core.FullTextSearch; +using Core.Infrastructure; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +class PostgreSqlAuditPersistence : BaseAuditPersistence, IPersistence +{ + readonly PostgreSqlAuditPersisterSettings settings; + + public PostgreSqlAuditPersistence(PostgreSqlAuditPersisterSettings settings) + { + this.settings = settings; + } + + public void AddPersistence(IServiceCollection services) + { + ConfigureDbContext(services); + RegisterDataStores(services, settings); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddHostedService(); + } + + public void AddInstaller(IServiceCollection services) + { + ConfigureDbContext(services); + RegisterDataStores(services, settings); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } + + void ConfigureDbContext(IServiceCollection services) + { + services.AddSingleton(settings); + services.AddSingleton(settings); + services.AddSingleton(settings); + + services.AddDbContext((serviceProvider, options) => + { + options.UseNpgsql(settings.ConnectionString, npgsqlOptions => + { + npgsqlOptions.CommandTimeout(settings.CommandTimeout); + if (settings.EnableRetryOnFailure) + { + npgsqlOptions.EnableRetryOnFailure( + maxRetryCount: 5, + maxRetryDelay: TimeSpan.FromSeconds(30), + errorCodesToAdd: null); + } + }); + + if (settings.EnableSensitiveDataLogging) + { + options.EnableSensitiveDataLogging(); + } + }, ServiceLifetime.Scoped); + + services.AddScoped(sp => sp.GetRequiredService()); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistenceConfiguration.cs new file mode 100644 index 0000000000..6746db7411 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersistenceConfiguration.cs @@ -0,0 +1,85 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +public class PostgreSqlAuditPersistenceConfiguration : IPersistenceConfiguration +{ + const string DatabaseConnectionStringKey = "Database/ConnectionString"; + const string CommandTimeoutKey = "Database/CommandTimeout"; + const string MessageBodyStoragePathKey = "MessageBody/StoragePath"; + const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize"; + const string StoreMessageBodiesOnDiskKey = "MessageBody/StoreOnDisk"; + const string MessageBodyStorageConnectionStringKey = "MessageBody/StorageConnectionString"; + + public string Name => "PostgreSQL"; + + public IEnumerable ConfigurationKeys => + [ + DatabaseConnectionStringKey, + CommandTimeoutKey, + MessageBodyStoragePathKey, + MinBodySizeForCompressionKey, + StoreMessageBodiesOnDiskKey, + MessageBodyStorageConnectionStringKey + ]; + + public IPersistence Create(PersistenceSettings settings) + { + var connectionString = GetRequiredSetting(settings, DatabaseConnectionStringKey); + + // Initialize message body storage path + var messageBodyStoragePath = GetSetting(settings, MessageBodyStoragePathKey, string.Empty); + var messageBodyStorageConnectionString = GetSetting(settings, MessageBodyStorageConnectionStringKey, string.Empty); + + var specificSettings = new PostgreSqlAuditPersisterSettings( + settings.AuditRetentionPeriod, + settings.EnableFullTextSearchOnBodies, + settings.MaxBodySizeToStore) + { + ConnectionString = connectionString, + MessageBodyStorageConnectionString = messageBodyStorageConnectionString, + CommandTimeout = GetSetting(settings, CommandTimeoutKey, 30), + MessageBodyStoragePath = messageBodyStoragePath, + MinBodySizeForCompression = GetSetting(settings, MinBodySizeForCompressionKey, 4096), + StoreMessageBodiesOnDisk = GetSetting(settings, StoreMessageBodiesOnDiskKey, true) + }; + + return new PostgreSqlAuditPersistence(specificSettings); + } + + static string GetRequiredSetting(PersistenceSettings settings, string key) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value; + } + + throw new Exception($"Setting {key} is required for PostgreSQL persistence. " + + $"Set environment variable: SERVICECONTROL_AUDIT_DATABASE_CONNECTIONSTRING"); + } + + static string GetSetting(PersistenceSettings settings, string key, string defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value; + } + return defaultValue; + } + + static int GetSetting(PersistenceSettings settings, string key, int defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && int.TryParse(value, out var intValue)) + { + return intValue; + } + return defaultValue; + } + + static bool GetSetting(PersistenceSettings settings, string key, bool defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && bool.TryParse(value, out var boolValue)) + { + return boolValue; + } + return defaultValue; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersisterSettings.cs b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersisterSettings.cs new file mode 100644 index 0000000000..e48ea5e7b4 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/PostgreSqlAuditPersisterSettings.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.Sql.PostgreSQL; + +using Core.Abstractions; + +public class PostgreSqlAuditPersisterSettings : AuditSqlPersisterSettings +{ + public PostgreSqlAuditPersisterSettings( + TimeSpan auditRetentionPeriod, + bool enableFullTextSearchOnBodies, + int maxBodySizeToStore) + : base(auditRetentionPeriod, enableFullTextSearchOnBodies, maxBodySizeToStore) + { + } + + public bool EnableRetryOnFailure { get; set; } = true; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/ServiceControl.Audit.Persistence.Sql.PostgreSQL.csproj b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/ServiceControl.Audit.Persistence.Sql.PostgreSQL.csproj new file mode 100644 index 0000000000..ec80ca3ac2 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/ServiceControl.Audit.Persistence.Sql.PostgreSQL.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/persistence.manifest b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/persistence.manifest new file mode 100644 index 0000000000..54a010cd1c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.PostgreSQL/persistence.manifest @@ -0,0 +1,25 @@ +{ + "Name": "PostgreSQL", + "DisplayName": "PostgreSQL", + "Description": "PostgreSQL ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.Sql.PostgreSQL", + "TypeName": "ServiceControl.Audit.Persistence.Sql.PostgreSQL.PostgreSqlAuditPersistenceConfiguration, ServiceControl.Audit.Persistence.Sql.PostgreSQL", + "Settings": [ + { + "Name": "ServiceControl.Audit/Database/ConnectionString", + "Mandatory": true + }, + { + "Name": "ServiceControl.Audit/Database/CommandTimeout", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/MessageBody/StoragePath", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/MessageBody/MinCompressionSize", + "Mandatory": false + } + ] +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/.editorconfig b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/.editorconfig new file mode 100644 index 0000000000..fc68ac3228 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/.editorconfig @@ -0,0 +1,9 @@ +[*.cs] + +# Justification: ServiceControl app has no synchronization context +dotnet_diagnostic.CA2007.severity = none + +# Disable style rules for auto-generated EF migrations +[Migrations/**.cs] +dotnet_diagnostic.IDE0065.severity = none +generated_code = true diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/KnownEndpointsReconciler.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/KnownEndpointsReconciler.cs new file mode 100644 index 0000000000..8a23ad9fb9 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/KnownEndpointsReconciler.cs @@ -0,0 +1,65 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Infrastructure; + +using Core.DbContexts; +using Core.Entities; +using Core.Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +class KnownEndpointsReconciler( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory) + : InsertOnlyTableReconciler( + logger, timeProvider, serviceScopeFactory, nameof(KnownEndpointsReconciler)) +{ + protected override async Task ReconcileBatch(AuditDbContextBase dbContext, CancellationToken stoppingToken) + { + var sql = @" + DECLARE @lockResult INT; + EXEC @lockResult = sp_getapplock @Resource = 'known_endpoints_sync', @LockMode = 'Exclusive', @LockOwner = 'Transaction', @LockTimeout = 0; + IF @lockResult < 0 + BEGIN + SELECT 0; + RETURN; + END; + + DECLARE @deleted TABLE ( + KnownEndpointId UNIQUEIDENTIFIER, + Name NVARCHAR(MAX), + HostId UNIQUEIDENTIFIER, + Host NVARCHAR(MAX), + LastSeen DATETIME2 + ); + + DELETE TOP (@batchSize) FROM KnownEndpointsInsertOnly + OUTPUT DELETED.KnownEndpointId, DELETED.Name, DELETED.HostId, DELETED.Host, DELETED.LastSeen + INTO @deleted; + + WITH ranked AS ( + SELECT KnownEndpointId, Name, HostId, Host, LastSeen, + ROW_NUMBER() OVER (PARTITION BY KnownEndpointId ORDER BY LastSeen DESC) AS rn + FROM @deleted + ), + aggregated AS ( + SELECT KnownEndpointId, Name, HostId, Host, LastSeen + FROM ranked + WHERE rn = 1 + ) + MERGE INTO KnownEndpoints AS target + USING aggregated AS source + ON target.Id = source.KnownEndpointId + WHEN MATCHED AND source.LastSeen > target.LastSeen THEN + UPDATE SET LastSeen = source.LastSeen + WHEN NOT MATCHED THEN + INSERT (Id, Name, HostId, Host, LastSeen) + VALUES (source.KnownEndpointId, source.Name, source.HostId, source.Host, source.LastSeen); + + SELECT @@ROWCOUNT; + "; + + var rowsAffected = await dbContext.Database.ExecuteSqlRawAsync(sql, [new Microsoft.Data.SqlClient.SqlParameter("@batchSize", BatchSize)], stoppingToken); + return rowsAffected; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/RetentionCleaner.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/RetentionCleaner.cs new file mode 100644 index 0000000000..3df58ac1d6 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/RetentionCleaner.cs @@ -0,0 +1,52 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Infrastructure; + +using System.Data.Common; +using Core.Abstractions; +using Core.Infrastructure; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +class RetentionCleaner( + ILogger logger, + TimeProvider timeProvider, + IServiceScopeFactory serviceScopeFactory, + AuditSqlPersisterSettings settings, + IBodyStoragePersistence bodyPersistence, + IPartitionManager partitionManager, + RetentionMetrics metrics) + : Core.Infrastructure.RetentionCleaner(logger, timeProvider, serviceScopeFactory, settings, bodyPersistence, partitionManager, metrics) +{ + readonly string connectionString = settings.ConnectionString; + + protected override DbConnection CreateConnection() => new SqlConnection(connectionString); + + protected override async Task TryAcquireLock(DbConnection lockConnection, CancellationToken stoppingToken) + { + await using var cmd = lockConnection.CreateCommand(); + cmd.CommandText = """ + DECLARE @lockResult INT; + EXEC @lockResult = sp_getapplock + @Resource = 'retention_cleaner', + @LockMode = 'Exclusive', + @LockOwner = 'Session', + @LockTimeout = 0; + SELECT @lockResult; + """; + + var result = await cmd.ExecuteScalarAsync(stoppingToken); + return result is int lockResult && lockResult >= 0; + } + + protected override async Task ReleaseLock(DbConnection lockConnection, CancellationToken stoppingToken) + { + await using var cmd = lockConnection.CreateCommand(); + cmd.CommandText = """ + EXEC sp_releaseapplock + @Resource = 'retention_cleaner', + @LockOwner = 'Session'; + """; + + await cmd.ExecuteNonQueryAsync(stoppingToken); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/SqlServerPartitionManager.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/SqlServerPartitionManager.cs new file mode 100644 index 0000000000..4cd8cc4dc5 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Infrastructure/SqlServerPartitionManager.cs @@ -0,0 +1,74 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Infrastructure; + +using Microsoft.EntityFrameworkCore; +using ServiceControl.Audit.Persistence.Sql.Core.DbContexts; +using ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; + +// Table names cannot be parameterized in SQL; all values come from internal constants +#pragma warning disable EF1002, EF1003 +public class SqlServerPartitionManager : IPartitionManager +{ + static readonly string[] Tables = ["ProcessedMessages", "SagaSnapshots"]; + + public Task EnsurePartitionsExist(AuditDbContextBase dbContext, DateTime currentHour, int hoursAhead, CancellationToken ct) + { + // No partitioning on SQL Server — nothing to prepare + return Task.CompletedTask; + } + + public async Task DropPartition(AuditDbContextBase dbContext, DateTime partitionHour, CancellationToken ct) + { + var hourStr = TruncateToHour(partitionHour).ToString("yyyy-MM-ddTHH:00:00"); + var nextHourStr = TruncateToHour(partitionHour).AddHours(1).ToString("yyyy-MM-ddTHH:00:00"); + + dbContext.Database.SetCommandTimeout(TimeSpan.FromMinutes(5)); + + foreach (var table in Tables) + { + while (true) + { + var deleted = await dbContext.Database.ExecuteSqlRawAsync( + "DELETE TOP (10000) FROM " + table + " WHERE CreatedOn >= '" + hourStr + "' AND CreatedOn < '" + nextHourStr + "'", ct); + + if (deleted == 0) + { + break; + } + } + } + } + + public async Task> GetExpiredPartitions(AuditDbContextBase dbContext, DateTime cutoff, CancellationToken ct) + { + var truncatedCutoff = TruncateToHour(cutoff); + + // Find the oldest hour that has data, then return all hourly buckets up to the cutoff + var oldestHours = await dbContext.Database + .SqlQueryRaw( + "SELECT MIN(CreatedOn) AS Value FROM ProcessedMessages " + + "UNION ALL " + + "SELECT MIN(CreatedOn) AS Value FROM SagaSnapshots") + .ToListAsync(ct); + + var oldest = oldestHours + .Where(d => d.HasValue) + .Select(d => TruncateToHour(d!.Value)) + .DefaultIfEmpty(truncatedCutoff) + .Min(); + + if (oldest >= truncatedCutoff) + { + return []; + } + + var hours = new List(); + for (var hour = oldest; hour < truncatedCutoff; hour = hour.AddHours(1)) + { + hours.Add(hour); + } + + return hours; + } + + static DateTime TruncateToHour(DateTime dt) => new(dt.Year, dt.Month, dt.Day, dt.Hour, 0, 0, dt.Kind); +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.Designer.cs new file mode 100644 index 0000000000..fa7c0a1685 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.Designer.cs @@ -0,0 +1,238 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.Sql.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + [Migration("20260214031615_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("FailedAuditImports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("KnownEndpointId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpointsInsertOnly", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("BodyNotStored") + .HasColumnType("bit"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("SearchableContent") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("ProcessedMessages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("SagaSnapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.cs new file mode 100644 index 0000000000..b5fa5cf8ad --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031615_InitialCreate.cs @@ -0,0 +1,170 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "FailedAuditImports", + columns: table => new + { + Id = table.Column(type: "uniqueidentifier", nullable: false), + MessageJson = table.Column(type: "nvarchar(max)", nullable: false), + ExceptionInfo = table.Column(type: "nvarchar(max)", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_FailedAuditImports", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "KnownEndpoints", + columns: table => new + { + Id = table.Column(type: "uniqueidentifier", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: false), + HostId = table.Column(type: "uniqueidentifier", nullable: false), + Host = table.Column(type: "nvarchar(max)", nullable: false), + LastSeen = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_KnownEndpoints", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "KnownEndpointsInsertOnly", + columns: table => new + { + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + KnownEndpointId = table.Column(type: "uniqueidentifier", nullable: false), + Name = table.Column(type: "nvarchar(max)", nullable: false), + HostId = table.Column(type: "uniqueidentifier", nullable: false), + Host = table.Column(type: "nvarchar(max)", nullable: false), + LastSeen = table.Column(type: "datetime2", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_KnownEndpointsInsertOnly", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "ProcessedMessages", + columns: table => new + { + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + CreatedOn = table.Column(type: "datetime2", nullable: false), + UniqueMessageId = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: false), + HeadersJson = table.Column(type: "nvarchar(max)", nullable: false), + SearchableContent = table.Column(type: "nvarchar(max)", nullable: true), + MessageId = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + MessageType = table.Column(type: "nvarchar(500)", maxLength: 500, nullable: true), + TimeSent = table.Column(type: "datetime2", nullable: true), + IsSystemMessage = table.Column(type: "bit", nullable: false), + Status = table.Column(type: "int", nullable: false), + ConversationId = table.Column(type: "nvarchar(200)", maxLength: 200, nullable: true), + ReceivingEndpointName = table.Column(type: "nvarchar(500)", maxLength: 500, nullable: true), + CriticalTimeTicks = table.Column(type: "bigint", nullable: true), + ProcessingTimeTicks = table.Column(type: "bigint", nullable: true), + DeliveryTimeTicks = table.Column(type: "bigint", nullable: true), + BodySize = table.Column(type: "int", nullable: false), + BodyUrl = table.Column(type: "nvarchar(500)", maxLength: 500, nullable: true), + BodyNotStored = table.Column(type: "bit", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ProcessedMessages", x => new { x.Id, x.CreatedOn }); + }); + + migrationBuilder.CreateTable( + name: "SagaSnapshots", + columns: table => new + { + Id = table.Column(type: "bigint", nullable: false) + .Annotation("SqlServer:Identity", "1, 1"), + CreatedOn = table.Column(type: "datetime2", nullable: false), + SagaId = table.Column(type: "uniqueidentifier", nullable: false), + SagaType = table.Column(type: "nvarchar(max)", nullable: false), + StartTime = table.Column(type: "datetime2", nullable: false), + FinishTime = table.Column(type: "datetime2", nullable: false), + Status = table.Column(type: "int", nullable: false), + StateAfterChange = table.Column(type: "nvarchar(max)", nullable: false), + InitiatingMessageJson = table.Column(type: "nvarchar(max)", nullable: false), + OutgoingMessagesJson = table.Column(type: "nvarchar(max)", nullable: false), + Endpoint = table.Column(type: "nvarchar(max)", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_SagaSnapshots", x => new { x.Id, x.CreatedOn }); + }); + + migrationBuilder.CreateIndex( + name: "IX_KnownEndpoints_LastSeen", + table: "KnownEndpoints", + column: "LastSeen"); + + migrationBuilder.CreateIndex( + name: "IX_KnownEndpointsInsertOnly_KnownEndpointId", + table: "KnownEndpointsInsertOnly", + column: "KnownEndpointId"); + + migrationBuilder.CreateIndex( + name: "IX_KnownEndpointsInsertOnly_LastSeen", + table: "KnownEndpointsInsertOnly", + column: "LastSeen"); + + migrationBuilder.CreateIndex( + name: "IX_ProcessedMessages_ConversationId", + table: "ProcessedMessages", + column: "ConversationId"); + + migrationBuilder.CreateIndex( + name: "IX_ProcessedMessages_MessageId", + table: "ProcessedMessages", + column: "MessageId"); + + migrationBuilder.CreateIndex( + name: "IX_ProcessedMessages_TimeSent", + table: "ProcessedMessages", + column: "TimeSent"); + + migrationBuilder.CreateIndex( + name: "IX_ProcessedMessages_UniqueMessageId", + table: "ProcessedMessages", + column: "UniqueMessageId"); + + migrationBuilder.CreateIndex( + name: "IX_SagaSnapshots_SagaId", + table: "SagaSnapshots", + column: "SagaId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "FailedAuditImports"); + + migrationBuilder.DropTable( + name: "KnownEndpoints"); + + migrationBuilder.DropTable( + name: "KnownEndpointsInsertOnly"); + + migrationBuilder.DropTable( + name: "ProcessedMessages"); + + migrationBuilder.DropTable( + name: "SagaSnapshots"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.Designer.cs new file mode 100644 index 0000000000..5b999792fd --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.Designer.cs @@ -0,0 +1,238 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.Sql.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + [Migration("20260214031634_AddFullTextSearch")] + partial class AddFullTextSearch + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("FailedAuditImports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("KnownEndpointId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpointsInsertOnly", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("BodyNotStored") + .HasColumnType("bit"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("SearchableContent") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("ProcessedMessages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("SagaSnapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.cs new file mode 100644 index 0000000000..8b74348a70 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214031634_AddFullTextSearch.cs @@ -0,0 +1,52 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + /// + public partial class AddFullTextSearch : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + CREATE UNIQUE NONCLUSTERED INDEX [UX_ProcessedMessages_FullTextKey] + ON [ProcessedMessages] ([Id]) + ON [PRIMARY]; + """); + + migrationBuilder.Sql(""" + IF NOT EXISTS (SELECT * FROM sys.fulltext_catalogs WHERE name = 'ProcessedMessagesCatalog') + BEGIN + CREATE FULLTEXT CATALOG ProcessedMessagesCatalog AS DEFAULT; + END + """, suppressTransaction: true); + + migrationBuilder.Sql(""" + CREATE FULLTEXT INDEX ON ProcessedMessages(SearchableContent LANGUAGE 0) + KEY INDEX UX_ProcessedMessages_FullTextKey + ON ProcessedMessagesCatalog + WITH STOPLIST = OFF; + """, suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + IF EXISTS (SELECT * FROM sys.fulltext_indexes WHERE object_id = OBJECT_ID('ProcessedMessages')) + BEGIN + DROP FULLTEXT INDEX ON ProcessedMessages; + END + """, suppressTransaction: true); + + migrationBuilder.Sql(""" + IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = OBJECT_ID('ProcessedMessages') AND name = 'UX_ProcessedMessages_FullTextKey') + BEGIN + DROP INDEX [UX_ProcessedMessages_FullTextKey] ON [ProcessedMessages]; + END + """); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.Designer.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.Designer.cs new file mode 100644 index 0000000000..ee8a24ecdc --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.Designer.cs @@ -0,0 +1,242 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.Sql.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + [Migration("20260214034809_AddCreatedOnIndexes")] + partial class AddCreatedOnIndexes + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("FailedAuditImports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("KnownEndpointId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpointsInsertOnly", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("BodyNotStored") + .HasColumnType("bit"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("SearchableContent") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("ProcessedMessages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("SagaSnapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.cs new file mode 100644 index 0000000000..44b7bec533 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/20260214034809_AddCreatedOnIndexes.cs @@ -0,0 +1,36 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + /// + public partial class AddCreatedOnIndexes : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_SagaSnapshots_CreatedOn", + table: "SagaSnapshots", + column: "CreatedOn"); + + migrationBuilder.CreateIndex( + name: "IX_ProcessedMessages_CreatedOn", + table: "ProcessedMessages", + column: "CreatedOn"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_SagaSnapshots_CreatedOn", + table: "SagaSnapshots"); + + migrationBuilder.DropIndex( + name: "IX_ProcessedMessages_CreatedOn", + table: "ProcessedMessages"); + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs new file mode 100644 index 0000000000..472b5144ac --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/Migrations/SqlServerAuditDbContextModelSnapshot.cs @@ -0,0 +1,239 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceControl.Audit.Persistence.Sql.SqlServer; + +#nullable disable + +namespace ServiceControl.Audit.Persistence.Sql.SqlServer.Migrations +{ + [DbContext(typeof(SqlServerAuditDbContext))] + partial class SqlServerAuditDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "10.0.3") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.FailedAuditImportEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("ExceptionInfo") + .HasColumnType("nvarchar(max)"); + + b.Property("MessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.ToTable("FailedAuditImports", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointEntity", b => + { + b.Property("Id") + .HasColumnType("uniqueidentifier"); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpoints", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.KnownEndpointInsertOnlyEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("Host") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("HostId") + .HasColumnType("uniqueidentifier"); + + b.Property("KnownEndpointId") + .HasColumnType("uniqueidentifier"); + + b.Property("LastSeen") + .HasColumnType("datetime2"); + + b.Property("Name") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.HasKey("Id"); + + b.HasIndex("KnownEndpointId"); + + b.HasIndex("LastSeen"); + + b.ToTable("KnownEndpointsInsertOnly", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.ProcessedMessageEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("BodyNotStored") + .HasColumnType("bit"); + + b.Property("BodySize") + .HasColumnType("int"); + + b.Property("BodyUrl") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ConversationId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("CriticalTimeTicks") + .HasColumnType("bigint"); + + b.Property("DeliveryTimeTicks") + .HasColumnType("bigint"); + + b.Property("HeadersJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("IsSystemMessage") + .HasColumnType("bit"); + + b.Property("MessageId") + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.Property("MessageType") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("ProcessingTimeTicks") + .HasColumnType("bigint"); + + b.Property("ReceivingEndpointName") + .HasMaxLength(500) + .HasColumnType("nvarchar(500)"); + + b.Property("SearchableContent") + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.Property("TimeSent") + .HasColumnType("datetime2"); + + b.Property("UniqueMessageId") + .IsRequired() + .HasMaxLength(200) + .HasColumnType("nvarchar(200)"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("ConversationId"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("MessageId"); + + b.HasIndex("TimeSent"); + + b.HasIndex("UniqueMessageId"); + + b.ToTable("ProcessedMessages", (string)null); + }); + + modelBuilder.Entity("ServiceControl.Audit.Persistence.Sql.Core.Entities.SagaSnapshotEntity", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("bigint"); + + SqlServerPropertyBuilderExtensions.UseIdentityColumn(b.Property("Id")); + + b.Property("CreatedOn") + .HasColumnType("datetime2"); + + b.Property("Endpoint") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("FinishTime") + .HasColumnType("datetime2"); + + b.Property("InitiatingMessageJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("OutgoingMessagesJson") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("SagaId") + .HasColumnType("uniqueidentifier"); + + b.Property("SagaType") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("StartTime") + .HasColumnType("datetime2"); + + b.Property("StateAfterChange") + .IsRequired() + .HasColumnType("nvarchar(max)"); + + b.Property("Status") + .HasColumnType("int"); + + b.HasKey("Id", "CreatedOn"); + + b.HasIndex("CreatedOn"); + + b.HasIndex("SagaId"); + + b.ToTable("SagaSnapshots", (string)null); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/ServiceControl.Audit.Persistence.Sql.SqlServer.csproj b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/ServiceControl.Audit.Persistence.Sql.SqlServer.csproj new file mode 100644 index 0000000000..4085f1a4a8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/ServiceControl.Audit.Persistence.Sql.SqlServer.csproj @@ -0,0 +1,26 @@ + + + + net10.0 + enable + enable + true + true + + + + + + + + + + + + + + + + + + diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDatabaseMigrator.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDatabaseMigrator.cs new file mode 100644 index 0000000000..f2db8dbe51 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDatabaseMigrator.cs @@ -0,0 +1,26 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Core.Abstractions; +using Core.DbContexts; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +class SqlServerAuditDatabaseMigrator( + AuditDbContextBase dbContext, + ILogger logger) + : IAuditDatabaseMigrator +{ + public async Task ApplyMigrations(CancellationToken cancellationToken = default) + { + logger.LogInformation("Starting SQL Server database migration for Audit"); + + var previousTimeout = dbContext.Database.GetCommandTimeout(); + dbContext.Database.SetCommandTimeout(TimeSpan.FromMinutes(40)); + + await dbContext.Database.MigrateAsync(cancellationToken).ConfigureAwait(false); + + dbContext.Database.SetCommandTimeout(previousTimeout); + + logger.LogInformation("SQL Server database migration completed for Audit"); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContext.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContext.cs new file mode 100644 index 0000000000..d41c3c982c --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContext.cs @@ -0,0 +1,20 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Core.DbContexts; +using Core.Entities; +using Microsoft.EntityFrameworkCore; + +public class SqlServerAuditDbContext : AuditDbContextBase +{ + public SqlServerAuditDbContext(DbContextOptions options) : base(options) + { + } + + protected override void OnModelCreatingProvider(ModelBuilder modelBuilder) + { + // SQL Server doesn't use native partitioning, so it needs an index on CreatedOn + // for efficient MIN() queries used by the retention cleaner + modelBuilder.Entity().HasIndex(e => e.CreatedOn); + modelBuilder.Entity().HasIndex(e => e.CreatedOn); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContextFactory.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContextFactory.cs new file mode 100644 index 0000000000..2fe2eba604 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditDbContextFactory.cs @@ -0,0 +1,24 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Design; + +/// +/// Design-time factory for EF Core migrations tooling. +/// +public class SqlServerAuditDbContextFactory : IDesignTimeDbContextFactory +{ + public SqlServerAuditDbContext CreateDbContext(string[] args) + { + var optionsBuilder = new DbContextOptionsBuilder(); + + // Use a default connection string for design-time operations + // This is only used when running EF Core migrations tooling + var connectionString = Environment.GetEnvironmentVariable("SERVICECONTROL_AUDIT_DATABASE_CONNECTIONSTRING") + ?? "Server=localhost;Database=ServiceControlAudit;Trusted_Connection=True;TrustServerCertificate=True;"; + + optionsBuilder.UseSqlServer(connectionString); + + return new SqlServerAuditDbContext(optionsBuilder.Options); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditFullTextSearchProvider.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditFullTextSearchProvider.cs new file mode 100644 index 0000000000..61cb663a8b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditFullTextSearchProvider.cs @@ -0,0 +1,17 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Core.Entities; +using Core.FullTextSearch; +using Microsoft.EntityFrameworkCore; + +class SqlServerAuditFullTextSearchProvider : IAuditFullTextSearchProvider +{ + public IQueryable ApplyFullTextSearch( + IQueryable query, + string searchTerms) + { + // Use SQL Server FREETEXT for natural language search on combined searchable content + return query.Where(pm => + EF.Functions.FreeText(pm.SearchableContent!, searchTerms)); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistence.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistence.cs new file mode 100644 index 0000000000..f0bae8a272 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistence.cs @@ -0,0 +1,68 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Core.Abstractions; +using Core.DbContexts; +using Core.FullTextSearch; +using Core.Infrastructure; +using Infrastructure; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; + +class SqlServerAuditPersistence : BaseAuditPersistence, IPersistence +{ + readonly SqlServerAuditPersisterSettings settings; + + public SqlServerAuditPersistence(SqlServerAuditPersisterSettings settings) + { + this.settings = settings; + } + + public void AddPersistence(IServiceCollection services) + { + ConfigureDbContext(services); + RegisterDataStores(services, settings); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + services.AddHostedService(); + services.AddHostedService(); + } + + public void AddInstaller(IServiceCollection services) + { + ConfigureDbContext(services); + RegisterDataStores(services, settings); + services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); + } + + void ConfigureDbContext(IServiceCollection services) + { + services.AddSingleton(settings); + services.AddSingleton(settings); + services.AddSingleton(settings); + + services.AddDbContext((serviceProvider, options) => + { + options.UseSqlServer(settings.ConnectionString, sqlOptions => + { + sqlOptions.CommandTimeout(settings.CommandTimeout); + if (settings.EnableRetryOnFailure) + { + sqlOptions.EnableRetryOnFailure( + maxRetryCount: 5, + maxRetryDelay: TimeSpan.FromSeconds(30), + errorNumbersToAdd: null); + } + }); + + if (settings.EnableSensitiveDataLogging) + { + options.EnableSensitiveDataLogging(); + } + }, ServiceLifetime.Scoped); + + services.AddScoped(sp => sp.GetRequiredService()); + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistenceConfiguration.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistenceConfiguration.cs new file mode 100644 index 0000000000..6421ccc4c8 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersistenceConfiguration.cs @@ -0,0 +1,84 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +public class SqlServerAuditPersistenceConfiguration : IPersistenceConfiguration +{ + const string DatabaseConnectionStringKey = "Database/ConnectionString"; + const string CommandTimeoutKey = "Database/CommandTimeout"; + const string MessageBodyStoragePathKey = "MessageBody/StoragePath"; + const string MinBodySizeForCompressionKey = "MessageBody/MinCompressionSize"; + const string StoreMessageBodiesOnDiskKey = "MessageBody/StoreOnDisk"; + const string MessageBodyStorageConnectionStringKey = "MessageBody/StorageConnectionString"; + public string Name => "SqlServer"; + + public IEnumerable ConfigurationKeys => + [ + DatabaseConnectionStringKey, + CommandTimeoutKey, + MessageBodyStoragePathKey, + MinBodySizeForCompressionKey, + StoreMessageBodiesOnDiskKey, + MessageBodyStorageConnectionStringKey + ]; + + public IPersistence Create(PersistenceSettings settings) + { + var connectionString = GetRequiredSetting(settings, DatabaseConnectionStringKey); + + // Initialize message body storage path + var messageBodyStoragePath = GetSetting(settings, MessageBodyStoragePathKey, string.Empty); + var messageBodyStorageConnectionString = GetSetting(settings, MessageBodyStorageConnectionStringKey, string.Empty); + + var specificSettings = new SqlServerAuditPersisterSettings( + settings.AuditRetentionPeriod, + settings.EnableFullTextSearchOnBodies, + settings.MaxBodySizeToStore) + { + ConnectionString = connectionString, + CommandTimeout = GetSetting(settings, CommandTimeoutKey, 30), + MessageBodyStoragePath = messageBodyStoragePath, + MessageBodyStorageConnectionString = messageBodyStorageConnectionString, + MinBodySizeForCompression = GetSetting(settings, MinBodySizeForCompressionKey, 4096), + StoreMessageBodiesOnDisk = GetSetting(settings, StoreMessageBodiesOnDiskKey, true) + }; + + return new SqlServerAuditPersistence(specificSettings); + } + + static string GetRequiredSetting(PersistenceSettings settings, string key) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value; + } + + throw new Exception($"Setting {key} is required for SQL Server persistence. " + + $"Set environment variable: SERVICECONTROL_AUDIT_DATABASE_CONNECTIONSTRING"); + } + + static string GetSetting(PersistenceSettings settings, string key, string defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value)) + { + return value; + } + return defaultValue; + } + + static int GetSetting(PersistenceSettings settings, string key, int defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && int.TryParse(value, out var intValue)) + { + return intValue; + } + return defaultValue; + } + + static bool GetSetting(PersistenceSettings settings, string key, bool defaultValue) + { + if (settings.PersisterSpecificSettings.TryGetValue(key, out var value) && bool.TryParse(value, out var boolValue)) + { + return boolValue; + } + return defaultValue; + } +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersisterSettings.cs b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersisterSettings.cs new file mode 100644 index 0000000000..78852f71aa --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/SqlServerAuditPersisterSettings.cs @@ -0,0 +1,16 @@ +namespace ServiceControl.Audit.Persistence.Sql.SqlServer; + +using Core.Abstractions; + +public class SqlServerAuditPersisterSettings : AuditSqlPersisterSettings +{ + public SqlServerAuditPersisterSettings( + TimeSpan auditRetentionPeriod, + bool enableFullTextSearchOnBodies, + int maxBodySizeToStore) + : base(auditRetentionPeriod, enableFullTextSearchOnBodies, maxBodySizeToStore) + { + } + + public bool EnableRetryOnFailure { get; set; } = true; +} diff --git a/src/ServiceControl.Audit.Persistence.Sql.SqlServer/persistence.manifest b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/persistence.manifest new file mode 100644 index 0000000000..328f31b701 --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Sql.SqlServer/persistence.manifest @@ -0,0 +1,25 @@ +{ + "Name": "SqlServer", + "DisplayName": "SQL Server", + "Description": "SQL Server ServiceControl Audit persister", + "AssemblyName": "ServiceControl.Audit.Persistence.Sql.SqlServer", + "TypeName": "ServiceControl.Audit.Persistence.Sql.SqlServer.SqlServerAuditPersistenceConfiguration, ServiceControl.Audit.Persistence.Sql.SqlServer", + "Settings": [ + { + "Name": "ServiceControl.Audit/Database/ConnectionString", + "Mandatory": true + }, + { + "Name": "ServiceControl.Audit/Database/CommandTimeout", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/MessageBody/StoragePath", + "Mandatory": false + }, + { + "Name": "ServiceControl.Audit/MessageBody/MinCompressionSize", + "Mandatory": false + } + ] +} diff --git a/src/ServiceControl.Audit.Persistence/PersistenceSettings.cs b/src/ServiceControl.Audit.Persistence/PersistenceSettings.cs index b5b51676de..807cba5f1d 100644 --- a/src/ServiceControl.Audit.Persistence/PersistenceSettings.cs +++ b/src/ServiceControl.Audit.Persistence/PersistenceSettings.cs @@ -23,6 +23,14 @@ public PersistenceSettings( public bool EnableFullTextSearchOnBodies { get; set; } + /// + /// Base path for storing message bodies on the filesystem. + /// Initialized by persistence configuration based on DatabasePath or explicit configuration. + /// + public string MessageBodyStoragePath { get; set; } + + public string MessageBodyStorageConnectionString { get; set; } + public int MaxBodySizeToStore { get; set; } public IDictionary PersisterSpecificSettings { get; } diff --git a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs index ea417c25a8..be91591a0b 100644 --- a/src/ServiceControl.Audit/Auditing/AuditIngestion.cs +++ b/src/ServiceControl.Audit/Auditing/AuditIngestion.cs @@ -2,6 +2,7 @@ { using System; using System.Collections.Generic; + using System.Linq; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -40,17 +41,27 @@ ILogger logger this.applicationLifetime = applicationLifetime; this.metrics = metrics; this.logger = logger; - if (!transportSettings.MaxConcurrency.HasValue) - { - throw new ArgumentException("MaxConcurrency is not set in TransportSettings"); - } - MaxBatchSize = transportSettings.MaxConcurrency.Value; + BatchSize = settings.AuditIngestionBatchSize; + MaxParallelWriters = settings.AuditIngestionMaxParallelWriters; + BatchTimeout = settings.AuditIngestionBatchTimeout; + + // Message channel: larger buffer to decouple transport from batch assembly + int messageChannelCapacity = BatchSize * MaxParallelWriters * 2; + messageChannel = Channel.CreateBounded(new BoundedChannelOptions(messageChannelCapacity) + { + SingleReader = true, // Batch assembler is single reader + SingleWriter = false, // Transport threads write concurrently + AllowSynchronousContinuations = false, + FullMode = BoundedChannelFullMode.Wait + }); - channel = Channel.CreateBounded(new BoundedChannelOptions(MaxBatchSize) + // Batch channel: holds assembled batches for parallel writers + int batchChannelCapacity = MaxParallelWriters * 2; + batchChannel = Channel.CreateBounded>(new BoundedChannelOptions(batchChannelCapacity) { - SingleReader = true, - SingleWriter = false, + SingleReader = false, // Multiple writers consume concurrently + SingleWriter = true, // Batch assembler is single writer AllowSynchronousContinuations = false, FullMode = BoundedChannelFullMode.Wait }); @@ -210,7 +221,7 @@ async Task OnMessage(MessageContext messageContext, CancellationToken cancellati var taskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); messageContext.SetTaskCompletionSource(taskCompletionSource); - await channel.Writer.WriteAsync(messageContext, cancellationToken); + await messageChannel.Writer.WriteAsync(messageContext, cancellationToken); _ = await taskCompletionSource.Task; messageIngestionMetrics.Success(); @@ -224,54 +235,145 @@ public override async Task StartAsync(CancellationToken cancellationToken) protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + // Start batch assembler task + // Note: Pass CancellationToken.None to Task.Run - if stoppingToken is already cancelled + // it would throw immediately without starting the task. Let the loop handle cancellation internally. + batchAssemblerTask = Task.Run(() => BatchAssemblerLoop(stoppingToken), CancellationToken.None); + + // Start parallel writer tasks + writerTasks = new Task[MaxParallelWriters]; + for (int i = 0; i < MaxParallelWriters; i++) + { + int writerId = i; + writerTasks[i] = Task.Run(() => WriterLoop(writerId, stoppingToken), CancellationToken.None); + } + try { - var contexts = new List(MaxBatchSize); + await Task.WhenAll(writerTasks.Append(batchAssemblerTask)); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Expected during shutdown + } + } - while (await channel.Reader.WaitToReadAsync(stoppingToken)) + async Task BatchAssemblerLoop(CancellationToken stoppingToken) + { + var batch = new List(BatchSize); + + try + { + while (!stoppingToken.IsCancellationRequested) { - // will only enter here if there is something to read. - try + // Wait for at least one message + if (!await messageChannel.Reader.WaitToReadAsync(stoppingToken)) + { + break; // Channel completed + } + + // Drain available messages up to BatchSize + while (batch.Count < BatchSize && messageChannel.Reader.TryRead(out var context)) + { + batch.Add(context); + } + + // If batch is not full, wait with timeout for more messages + if (batch.Count > 0 && batch.Count < BatchSize) { - using var batchMetrics = metrics.BeginBatch(MaxBatchSize); + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + timeoutCts.CancelAfter(BatchTimeout); - // as long as there is something to read this will fetch up to MaximumConcurrency items - while (channel.Reader.TryRead(out var context)) + try { - contexts.Add(context); + while (batch.Count < BatchSize) + { + if (!await messageChannel.Reader.WaitToReadAsync(timeoutCts.Token)) + { + break; // Channel completed + } + + while (batch.Count < BatchSize && messageChannel.Reader.TryRead(out var context)) + { + batch.Add(context); + } + } } + catch (OperationCanceledException) when (!stoppingToken.IsCancellationRequested) + { + // Timeout reached, dispatch partial batch + } + } - await auditIngestor.Ingest(contexts, stoppingToken); + if (batch.Count > 0) + { + await batchChannel.Writer.WriteAsync(batch, stoppingToken); + batch = new List(BatchSize); + } + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Expected during shutdown + } + finally + { + // Complete the batch channel to signal writers to finish + batchChannel.Writer.Complete(); - batchMetrics.Complete(contexts.Count); + // Cancel any remaining messages in incomplete batch + foreach (var context in batch) + { + _ = context.GetTaskCompletionSource().TrySetCanceled(stoppingToken); + } + } + } + + async Task WriterLoop(int writerId, CancellationToken stoppingToken) + { + logger.LogDebug("Writer {WriterId} started", writerId); + List currentBatch = null; + + try + { + await foreach (var batch in batchChannel.Reader.ReadAllAsync(stoppingToken)) + { + currentBatch = batch; + try + { + using var batchMetrics = metrics.BeginBatch(BatchSize); + + await auditIngestor.Ingest(currentBatch, stoppingToken); + + batchMetrics.Complete(currentBatch.Count); + currentBatch = null; // Successfully processed } - catch (Exception e) + catch (Exception e) when (e is not OperationCanceledException) { - // signal all message handling tasks to terminate - foreach (var context in contexts) + // Signal failure to all messages in this batch + foreach (var context in currentBatch) { _ = context.GetTaskCompletionSource().TrySetException(e); } - if (e is OperationCanceledException && stoppingToken.IsCancellationRequested) - { - logger.LogInformation(e, "Batch cancelled"); - break; - } - - logger.LogInformation(e, "Ingesting messages failed"); - } - finally - { - contexts.Clear(); + currentBatch = null; // Failure handled + logger.LogWarning(e, "Writer {WriterId} failed to ingest batch", writerId); } } - // will fall out here when writer is completed } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { - // ExecuteAsync cancelled + // Expected during shutdown - signal cancellation for any in-flight batch + if (currentBatch != null) + { + foreach (var context in currentBatch) + { + _ = context.GetTaskCompletionSource().TrySetCanceled(stoppingToken); + } + } } + + logger.LogDebug("Writer {WriterId} stopped", writerId); } public override async Task StopAsync(CancellationToken cancellationToken) @@ -279,9 +381,28 @@ public override async Task StopAsync(CancellationToken cancellationToken) try { await watchdog.Stop(cancellationToken); - channel.Writer.Complete(); + + // Complete message channel to stop accepting new messages + messageChannel.Writer.Complete(); + + // Wait for batch assembler to finish (it completes the batch channel) + if (batchAssemblerTask != null) + { + await batchAssemblerTask.WaitAsync(cancellationToken); + } + + // Wait for all writers to finish + if (writerTasks != null) + { + await Task.WhenAll(writerTasks).WaitAsync(cancellationToken); + } + await base.StopAsync(cancellationToken); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + logger.LogInformation("Graceful shutdown timed out"); + } finally { if (transportInfrastructure != null) @@ -298,10 +419,20 @@ public override async Task StopAsync(CancellationToken cancellationToken) } } + public override void Dispose() + { + startStopSemaphore.Dispose(); + base.Dispose(); + } + TransportInfrastructure transportInfrastructure; IMessageReceiver messageReceiver; + Task batchAssemblerTask; + Task[] writerTasks; - readonly int MaxBatchSize; + readonly int BatchSize; + readonly int MaxParallelWriters; + readonly TimeSpan BatchTimeout; readonly SemaphoreSlim startStopSemaphore = new(1); readonly string inputEndpoint; readonly ITransportCustomization transportCustomization; @@ -310,7 +441,8 @@ public override async Task StopAsync(CancellationToken cancellationToken) readonly AuditIngestionFaultPolicy errorHandlingPolicy; readonly IAuditIngestionUnitOfWorkFactory unitOfWorkFactory; readonly Settings settings; - readonly Channel channel; + readonly Channel messageChannel; + readonly Channel> batchChannel; readonly Watchdog watchdog; readonly IHostApplicationLifetime applicationLifetime; readonly IngestionMetrics metrics; diff --git a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs index f8bd2d763a..9cdd97e0e8 100644 --- a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs +++ b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetrics.cs @@ -11,15 +11,15 @@ public class IngestionMetrics { public const string MeterName = "Particular.ServiceControl.Audit"; - public static readonly string BatchDurationInstrumentName = $"{InstrumentPrefix}.batch_duration_seconds"; - public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration_seconds"; + public static readonly string BatchDurationInstrumentName = $"{InstrumentPrefix}.batch_duration"; + public static readonly string MessageDurationInstrumentName = $"{InstrumentPrefix}.message_duration"; public IngestionMetrics(IMeterFactory meterFactory) { var meter = meterFactory.Create(MeterName, MeterVersion); - batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "seconds", "Message batch processing duration in seconds"); - ingestionDuration = meter.CreateHistogram(MessageDurationInstrumentName, unit: "seconds", description: "Audit message processing duration in seconds"); + batchDuration = meter.CreateHistogram(BatchDurationInstrumentName, unit: "s", "Message batch processing duration in seconds"); + ingestionDuration = meter.CreateHistogram(MessageDurationInstrumentName, unit: "s", description: "Audit message processing duration in seconds"); consecutiveBatchFailureGauge = meter.CreateObservableGauge($"{InstrumentPrefix}.consecutive_batch_failures_total", () => consecutiveBatchFailures, description: "Consecutive audit ingestion batch failures"); failureCounter = meter.CreateCounter($"{InstrumentPrefix}.failures_total", description: "Audit ingestion failure count"); } @@ -72,4 +72,4 @@ void RecordBatchOutcome(bool success) const string InstrumentPrefix = "sc.audit.ingestion"; static readonly string SagaUpdateMessageType = typeof(SagaUpdatedMessage).FullName; -} \ No newline at end of file +} diff --git a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetricsConfiguration.cs b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetricsConfiguration.cs index b78b6cd62a..e5ca7fa4f9 100644 --- a/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetricsConfiguration.cs +++ b/src/ServiceControl.Audit/Auditing/Metrics/IngestionMetricsConfiguration.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Audit.Auditing.Metrics; using OpenTelemetry.Metrics; +using ServiceControl.Audit.Persistence.Sql.Core.Infrastructure; public static class IngestionMetricsConfiguration { @@ -15,5 +16,10 @@ public static void AddIngestionMetrics(this MeterProviderBuilder builder) builder.AddView( instrumentName: IngestionMetrics.BatchDurationInstrumentName, new ExplicitBucketHistogramConfiguration { Boundaries = [0.01, 0.05, 0.1, 0.5, 1, 5] }); + + // Retention cleanup metrics - using longer bucket boundaries since cleanup operations take longer + builder.AddView( + instrumentName: RetentionMetrics.CleanupDurationInstrumentName, + new ExplicitBucketHistogramConfiguration { Boundaries = [1, 5, 10, 30, 60, 300] }); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit/Infrastructure/Hosting/Commands/SetupCommand.cs b/src/ServiceControl.Audit/Infrastructure/Hosting/Commands/SetupCommand.cs index 73480e72e2..fd66ae60f5 100644 --- a/src/ServiceControl.Audit/Infrastructure/Hosting/Commands/SetupCommand.cs +++ b/src/ServiceControl.Audit/Infrastructure/Hosting/Commands/SetupCommand.cs @@ -1,10 +1,15 @@ namespace ServiceControl.Audit.Infrastructure.Hosting.Commands { + using System; using System.Collections.Generic; + using System.IO; using System.Runtime.InteropServices; using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; + using ServiceControl.Audit.Persistence; + using ServiceControl.Audit.Persistence.Sql.Core.Abstractions; using ServiceControl.Infrastructure; using Settings; using Transports; @@ -46,6 +51,25 @@ public override async Task Execute(HostArguments args, Settings settings) using var host = hostBuilder.Build(); await host.StartAsync(); + + if (settings.IngestAuditMessages) + { + // Create message body storage directory if it doesn't exist + var persistenceSettings = host.Services.GetRequiredService(); + if (!string.IsNullOrEmpty(persistenceSettings.MessageBodyStoragePath)) + { + if (!Directory.Exists(persistenceSettings.MessageBodyStoragePath)) + { + Directory.CreateDirectory(persistenceSettings.MessageBodyStoragePath); + } + } + else if (string.IsNullOrEmpty(persistenceSettings.MessageBodyStorageConnectionString)) + { + throw new Exception("Message body storage path is not configured."); + } + } + await host.Services.GetRequiredService().ApplyMigrations(); + await host.StopAsync(); } } diff --git a/src/ServiceControl.Audit/Infrastructure/NServiceBusFactory.cs b/src/ServiceControl.Audit/Infrastructure/NServiceBusFactory.cs index a6cd60b3de..1c9f040ad0 100644 --- a/src/ServiceControl.Audit/Infrastructure/NServiceBusFactory.cs +++ b/src/ServiceControl.Audit/Infrastructure/NServiceBusFactory.cs @@ -46,10 +46,10 @@ public static void Configure(Settings.Settings settings, ITransportCustomization routing.RouteToEndpoint(typeof(RegisterNewEndpoint), serviceControlLogicalQueue); routing.RouteToEndpoint(typeof(MarkMessageFailureResolvedByRetry), serviceControlLogicalQueue); - configuration.ReportCustomChecksTo( - transportCustomization.ToTransportQualifiedQueueName(settings.ServiceControlQueueAddress), - TimeSpan.FromMinutes(1) // Prevent clock skew issues, overrides calculated TTL due to some custom check using short reporting intervals (i.e. 5s results in 20s TTL) - ); + // configuration.ReportCustomChecksTo( + // transportCustomization.ToTransportQualifiedQueueName(settings.ServiceControlQueueAddress), + // TimeSpan.FromMinutes(1) // Prevent clock skew issues, overrides calculated TTL due to some custom check using short reporting intervals (i.e. 5s results in 20s TTL) + // ); } configuration.GetSettings().Set(settings.LoggingSettings); diff --git a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs index 3203bd349e..39187b6d99 100644 --- a/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs +++ b/src/ServiceControl.Audit/Infrastructure/Settings/Settings.cs @@ -53,6 +53,9 @@ public Settings(string transportType = null, string persisterType = null, Loggin MaximumConcurrencyLevel = SettingsReader.Read(SettingsRootNamespace, "MaximumConcurrencyLevel"); ServiceControlQueueAddress = SettingsReader.Read(SettingsRootNamespace, "ServiceControlQueueAddress"); TimeToRestartAuditIngestionAfterFailure = GetTimeToRestartAuditIngestionAfterFailure(); + AuditIngestionBatchSize = GetAuditIngestionBatchSize(); + AuditIngestionMaxParallelWriters = GetAuditIngestionMaxParallelWriters(); + AuditIngestionBatchTimeout = GetAuditIngestionBatchTimeout(); EnableFullTextSearchOnBodies = SettingsReader.Read(SettingsRootNamespace, "EnableFullTextSearchOnBodies", true); ShutdownTimeout = SettingsReader.Read(SettingsRootNamespace, "ShutdownTimeout", ShutdownTimeout); @@ -185,6 +188,9 @@ public int MaxBodySizeToStore public TimeSpan TimeToRestartAuditIngestionAfterFailure { get; set; } + public int AuditIngestionBatchSize { get; set; } + public int AuditIngestionMaxParallelWriters { get; set; } + public TimeSpan AuditIngestionBatchTimeout { get; set; } public bool EnableFullTextSearchOnBodies { get; set; } // The default value is set to the maximum allowed time by the most @@ -315,6 +321,80 @@ static string Subscope(string address) return $"{queue}.log@{machine}"; } + int GetAuditIngestionBatchSize() + { + var value = SettingsReader.Read(SettingsRootNamespace, "AuditIngestionBatchSize", 50); + + if (ValidateConfiguration && value < 1) + { + var message = $"{nameof(AuditIngestionBatchSize)} setting is invalid, minimum value is 1."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + if (ValidateConfiguration && value > 500) + { + var message = $"{nameof(AuditIngestionBatchSize)} setting is invalid, maximum value is 500."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + return value; + } + + int GetAuditIngestionMaxParallelWriters() + { + var value = SettingsReader.Read(SettingsRootNamespace, "AuditIngestionMaxParallelWriters", 4); + + if (ValidateConfiguration && value < 1) + { + var message = $"{nameof(AuditIngestionMaxParallelWriters)} setting is invalid, minimum value is 1."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + if (ValidateConfiguration && value > 16) + { + var message = $"{nameof(AuditIngestionMaxParallelWriters)} setting is invalid, maximum value is 16."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + return value; + } + + TimeSpan GetAuditIngestionBatchTimeout() + { + var valueRead = SettingsReader.Read(SettingsRootNamespace, "AuditIngestionBatchTimeout"); + if (valueRead == null) + { + return TimeSpan.FromMilliseconds(100); + } + + if (TimeSpan.TryParse(valueRead, out var result)) + { + if (ValidateConfiguration && result < TimeSpan.FromMilliseconds(10)) + { + var message = $"{nameof(AuditIngestionBatchTimeout)} setting is invalid, minimum value is 10 milliseconds."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + if (ValidateConfiguration && result > TimeSpan.FromSeconds(5)) + { + var message = $"{nameof(AuditIngestionBatchTimeout)} setting is invalid, maximum value is 5 seconds."; + InternalLogger.Fatal(message); + throw new Exception(message); + } + + return result; + } + + var parseMessage = $"{nameof(AuditIngestionBatchTimeout)} setting is invalid, please make sure it is a TimeSpan."; + InternalLogger.Fatal(parseMessage); + throw new Exception(parseMessage); + } + // logger is intentionally not static to prevent it from being initialized before LoggingConfigurator.ConfigureLogging has been called readonly ILogger logger = LoggerUtil.CreateStaticLogger(); public const string DEFAULT_INSTANCE_NAME = "Particular.ServiceControl.Audit"; diff --git a/src/ServiceControl.Audit/Persistence/PersistenceConfigurationFactory.cs b/src/ServiceControl.Audit/Persistence/PersistenceConfigurationFactory.cs index d79ee76fe0..5ebad72eae 100644 --- a/src/ServiceControl.Audit/Persistence/PersistenceConfigurationFactory.cs +++ b/src/ServiceControl.Audit/Persistence/PersistenceConfigurationFactory.cs @@ -1,27 +1,21 @@ namespace ServiceControl.Audit.Persistence { using System; - using System.IO; using Configuration; using ServiceControl.Audit.Infrastructure.Settings; + using ServiceControl.Audit.Persistence.Sql.PostgreSQL; + using ServiceControl.Audit.Persistence.Sql.SqlServer; static class PersistenceConfigurationFactory { public static IPersistenceConfiguration LoadPersistenceConfiguration(Settings settings) { - try + return settings.PersistenceType switch { - var persistenceManifest = PersistenceManifestLibrary.Find(settings.PersistenceType); - var assemblyPath = Path.Combine(persistenceManifest.Location, $"{persistenceManifest.AssemblyName}.dll"); - var loadContext = settings.AssemblyLoadContextResolver(assemblyPath); - var customizationType = Type.GetType(persistenceManifest.TypeName, loadContext.LoadFromAssemblyName, null, true); - - return (IPersistenceConfiguration)Activator.CreateInstance(customizationType); - } - catch (Exception e) - { - throw new Exception($"Could not load persistence customization type {settings.PersistenceType}.", e); - } + "PostgreSQL" => new PostgreSqlAuditPersistenceConfiguration(), + "SqlServer" => new SqlServerAuditPersistenceConfiguration(), + _ => throw new Exception($"Unsupported persistence type {settings.PersistenceType}."), + }; } public static PersistenceSettings BuildPersistenceSettings(this IPersistenceConfiguration persistenceConfiguration, Settings settings) diff --git a/src/ServiceControl.Audit/ServiceControl.Audit.csproj b/src/ServiceControl.Audit/ServiceControl.Audit.csproj index 1752bf81bd..048d0d5511 100644 --- a/src/ServiceControl.Audit/ServiceControl.Audit.csproj +++ b/src/ServiceControl.Audit/ServiceControl.Audit.csproj @@ -11,10 +11,11 @@ - + + diff --git a/src/ServiceControl.Infrastructure/LoggingConfigurator.cs b/src/ServiceControl.Infrastructure/LoggingConfigurator.cs index f86f776bac..9ca30ee6c5 100644 --- a/src/ServiceControl.Infrastructure/LoggingConfigurator.cs +++ b/src/ServiceControl.Infrastructure/LoggingConfigurator.cs @@ -71,8 +71,15 @@ public static string ConfigureNLog(string logFileName, string logPath, LogLevel FinalMinLevel = LogLevel.Warn }; + var efCoreRule = new LoggingRule() + { + LoggerNamePattern = "Microsoft.EntityFrameworkCore.*", + FinalMinLevel = LogLevel.Warn + }; + nlogConfig.LoggingRules.Add(aspNetCoreRule); nlogConfig.LoggingRules.Add(httpClientRule); + nlogConfig.LoggingRules.Add(efCoreRule); nlogConfig.LoggingRules.Add(new LoggingRule("*", logLevel, consoleTarget)); diff --git a/src/ServiceControl.Persistence/IDatabaseMigrator.cs b/src/ServiceControl.Persistence/IDatabaseMigrator.cs new file mode 100644 index 0000000000..d5836fe503 --- /dev/null +++ b/src/ServiceControl.Persistence/IDatabaseMigrator.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Persistence; + +using System.Threading; +using System.Threading.Tasks; + +public interface IDatabaseMigrator +{ + Task ApplyMigrations(CancellationToken cancellationToken = default); +} diff --git a/src/ServiceControl.slnx b/src/ServiceControl.slnx index 2ff353b79a..08a9be0410 100644 --- a/src/ServiceControl.slnx +++ b/src/ServiceControl.slnx @@ -13,6 +13,7 @@ + @@ -20,6 +21,9 @@ + + +