.Net: Switch MEVD MongoDB Driver to v3.51 (#13370)

### Motivation and Context

Switches the MongoDB Driver to the latest 3.5 release which had a number
of breaking changes.

This affects both the MongoDB vector data project and the CosmosMongoDB
project.

This fixes #11652 and likely addresses  #12707 and partly #10291.

### Description

Switches the MongoDB driver to 3.5. Part of the breaking changes is that
GUIDs in BSON no longer have a default storage format specified due to
the need to switch from the C#-only format to the cross-MongoDB-driver
standard format. Setting this is achieved in this PR by way of both a
convention for the registry based mode and an alternative to
BsonValue.Create in the scenarios where we don't have access to
conventions/serialization such as key creation and using the dynamic
mapper.

### Contribution Checklist

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄


cc @roji

---------

Co-authored-by: Arthur Vickers <ajcvickers@hotmail.com>
Co-authored-by: Mark Wallace <127216156+markwallace-microsoft@users.noreply.github.com>
Co-authored-by: westey <164392973+westey-m@users.noreply.github.com>
Co-authored-by: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com>
Co-authored-by: Shay Rojansky <roji@roji.org>

Author: 6 people

dotnet/Directory.Packages.props

@@ -161,7 +161,7 @@
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="9.0.10" />
     <PackageVersion Include="DuckDB.NET.Data.Full" Version="1.2.0" />
     <PackageVersion Include="DuckDB.NET.Data" Version="1.1.3" />
-    <PackageVersion Include="MongoDB.Driver" Version="2.30.0" />
+    <PackageVersion Include="MongoDB.Driver" Version="3.5.2" />
     <PackageVersion Include="Microsoft.Graph" Version="5.94.0" />
     <PackageVersion Include="Microsoft.OpenApi" Version="1.6.24" />
     <PackageVersion Include="Microsoft.OpenApi.Readers" Version="1.6.24" />

dotnet/src/IntegrationTests/Connectors/Memory/CosmosMongoDB/CosmosMongoVectorStoreFixture.cs

@@ -40,7 +40,9 @@ public CosmosMongoVectorStoreFixture()
             .Build();
 
         var connectionString = GetConnectionString(configuration);
+#pragma warning disable CA2000 // Dispose objects before losing scope
         var client = new MongoClient(connectionString);
+#pragma warning restore CA2000
 
         this.MongoDatabase = client.GetDatabase("test");
 

dotnet/src/IntegrationTests/Connectors/Memory/MongoDB/MongoDBVectorStoreFixture.cs

@@ -37,11 +37,13 @@ public async Task InitializeAsync()
         cts.CancelAfter(TimeSpan.FromSeconds(60));
         await this._container.StartAsync(cts.Token);
 
+#pragma warning disable CA2000 // Dispose objects before losing scope
         var mongoClient = new MongoClient(new MongoClientSettings
         {
             Server = new MongoServerAddress(this._container.Hostname, this._container.GetMappedPublicPort(MongoDbBuilder.MongoDbPort)),
             DirectConnection = true,
         });
+#pragma warning restore CA2000
 
         this.MongoDatabase = mongoClient.GetDatabase("test");
 

dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs

@@ -0,0 +1,30 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using MongoDB.Bson;
+
+namespace Microsoft.SemanticKernel.Connectors.MongoDB;
+
+/// <summary>
+/// A class that constructs the correct BsonValue for a given CLR type.
+/// </summary>
+internal static class BsonValueFactory
+{
+    /// <summary>
+    /// Create a BsonValue for the given CLR type.
+    /// </summary>
+    /// <param name="value">The CLR object to create a BSON value for.</param>
+    /// <returns>The appropriate <see cref="BsonValue"/> for that CLR type.</returns>
+    public static BsonValue Create(object? value)
+        => value switch
+        {
+            null => BsonNull.Value,
+            Guid guid => new BsonBinaryData(guid, GuidRepresentation.Standard),
+            object[] array => new BsonArray(Array.ConvertAll(array, Create)),
+            Array array => new BsonArray(array),
+            IEnumerable<object> enumerable => new BsonArray(enumerable.Select(Create)),
+            _ => BsonValue.Create(value)
+        };
+}

dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs

@@ -30,21 +30,21 @@ public BsonDocument MapFromDataToStorageModel(Dictionary<string, object?> dataMo
             : keyValue switch
             {
                 string s => s,
-                Guid g => BsonValue.Create(g),
+                Guid g => new BsonBinaryData(g, GuidRepresentation.Standard),
                 ObjectId o => o,
                 long i => i,
                 int i => i,
 
                 null => throw new InvalidOperationException($"Key property '{model.KeyProperty.ModelName}' is null."),
-                _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string.")
+                _ => throw new InvalidCastException($"Key property '{model.KeyProperty.ModelName}' must be a string, Guid, ObjectID, long or int.")
             }
         };
 
         foreach (var property in model.DataProperties)
         {
             if (dataModel.TryGetValue(property.ModelName, out var dataValue))
             {
-                document[property.StorageName] = BsonValue.Create(dataValue);
+                document[property.StorageName] = BsonValueFactory.Create(dataValue);
             }
         }
 

dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoMapper.cs

@@ -11,6 +11,7 @@
 using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Bson.Serialization.Conventions;
+using MongoDB.Bson.Serialization.Serializers;
 
 namespace Microsoft.SemanticKernel.Connectors.MongoDB;
 
@@ -40,7 +41,8 @@ public MongoMapper(CollectionModel model)
 
         var conventionPack = new ConventionPack
         {
-            new IgnoreExtraElementsConvention(ignoreExtraElements: true)
+            new IgnoreExtraElementsConvention(ignoreExtraElements: true),
+            new GuidStandardRepresentationConvention()
         };
 
         ConventionRegistry.Register(
@@ -139,4 +141,15 @@ public TRecord MapFromStorageToDataModel(BsonDocument storageModel, bool include
 
         return BsonSerializer.Deserialize<TRecord>(storageModel);
     }
+
+    private class GuidStandardRepresentationConvention : ConventionBase, IMemberMapConvention
+    {
+        public void Apply(BsonMemberMap memberMap)
+        {
+            if (memberMap.MemberType == typeof(Guid) && memberMap.MemberInfo.GetCustomAttribute<BsonRepresentationAttribute>() is null)
+            {
+                memberMap.SetSerializer(new GuidSerializer(GuidRepresentation.Standard));
+            }
+        }
+    }
 }

dotnet/src/InternalUtilities/src/Diagnostics/NullableAttributes.cs

@@ -7,9 +7,10 @@
 // This was copied from https://github.com/dotnet/runtime/blob/39b9607807f29e48cae4652cd74735182b31182e/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/NullableAttributes.cs
 // and updated to have the scope of the attributes be internal.
 
-#if !NETCOREAPP
 namespace System.Diagnostics.CodeAnalysis;
 
+#if !NETCOREAPP && !NETSTANDARD2_1
+
 /// <summary>Specifies that null is allowed as an input even if the corresponding type disallows it.</summary>
 [AttributeUsage(AttributeTargets.Field | AttributeTargets.Parameter | AttributeTargets.Property, Inherited = false)]
 internal sealed class AllowNullAttribute : Attribute

dotnet/src/InternalUtilities/src/Http/HttpClientProvider.cs

@@ -88,7 +88,7 @@ private static SocketsHttpHandler CreateHandler()
                 },
             };
         }
-#elif NETSTANDARD2_0
+#elif NETSTANDARD2_0_OR_GREATER
         private static HttpClientHandler CreateHandler()
         {
             var handler = new HttpClientHandler();
@@ -99,7 +99,7 @@ private static HttpClientHandler CreateHandler()
             catch (PlatformNotSupportedException) { } // not supported on older frameworks
             return handler;
         }
-#elif NET462
+#elif NETFRAMEWORK
         private static HttpClientHandler CreateHandler()
             => new();
 #endif

dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollection.cs

@@ -15,6 +15,7 @@
 using Microsoft.Extensions.VectorData.ProviderServices;
 using Microsoft.SemanticKernel.Connectors.MongoDB;
 using MongoDB.Bson;
+using MongoDB.Bson.Serialization;
 using MongoDB.Driver;
 using MEVD = Microsoft.Extensions.VectorData;
 
@@ -67,6 +68,9 @@ public class CosmosMongoCollection<TKey, TRecord> : VectorStoreCollection<TKey,
     /// <summary>The size of the dynamic candidate list for search.</summary>
     private readonly int _efSearch;
 
+    /// <summary><see cref="BsonSerializationInfo"/> to use for serializing key values.</summary>
+    private readonly BsonSerializationInfo? _keySerializationInfo;
+
     private static readonly Type[] s_validKeyTypes = [typeof(string), typeof(Guid), typeof(ObjectId), typeof(int), typeof(long)];
 
     /// <summary>
@@ -123,6 +127,11 @@ internal CosmosMongoCollection(IMongoDatabase mongoDatabase, string name, Func<C
             VectorStoreName = mongoDatabase.DatabaseNamespace?.DatabaseName,
             CollectionName = name
         };
+
+        // Cache the key serialization info if possible
+        this._keySerializationInfo = typeof(TKey) == typeof(object)
+            ? null
+            : this.GetKeySerializationInfo();
     }
 
     /// <inheritdoc />
@@ -142,9 +151,9 @@ await this.RunOperationAsync("CreateIndexes",
     /// <inheritdoc />
     public override async Task DeleteAsync(TKey key, CancellationToken cancellationToken = default)
     {
-        var stringKey = this.GetStringKey(key);
+        Verify.NotNull(key);
 
-        await this.RunOperationAsync("DeleteOne", () => this._mongoCollection.DeleteOneAsync(this.GetFilterById(stringKey), cancellationToken))
+        await this.RunOperationAsync("DeleteOne", () => this._mongoCollection.DeleteOneAsync(this.GetFilterById(key), cancellationToken))
             .ConfigureAwait(false);
     }
 
@@ -153,9 +162,7 @@ public override async Task DeleteAsync(IEnumerable<TKey> keys, CancellationToken
     {
         Verify.NotNull(keys);
 
-        var stringKeys = keys is IEnumerable<string> k ? k : keys.Cast<string>();
-
-        await this.RunOperationAsync("DeleteMany", () => this._mongoCollection.DeleteManyAsync(this.GetFilterByIds(stringKeys), cancellationToken))
+        await this.RunOperationAsync("DeleteMany", () => this._mongoCollection.DeleteManyAsync(this.GetFilterByIds(keys), cancellationToken))
             .ConfigureAwait(false);
     }
 
@@ -166,7 +173,7 @@ public override Task EnsureCollectionDeletedAsync(CancellationToken cancellation
     /// <inheritdoc />
     public override async Task<TRecord?> GetAsync(TKey key, RecordRetrievalOptions? options = null, CancellationToken cancellationToken = default)
     {
-        var stringKey = this.GetStringKey(key);
+        Verify.NotNull(key);
 
         var includeVectors = options?.IncludeVectors ?? false;
         if (includeVectors && this._model.EmbeddingGenerationRequired)
@@ -175,7 +182,7 @@ public override Task EnsureCollectionDeletedAsync(CancellationToken cancellation
         }
 
         using var cursor = await this
-            .FindAsync(this.GetFilterById(stringKey), top: 1, skip: null, includeVectors, sortDefinition: null, cancellationToken)
+            .FindAsync(this.GetFilterById(key), top: 1, skip: null, includeVectors, sortDefinition: null, cancellationToken)
             .ConfigureAwait(false);
 
         var record = await cursor.SingleOrDefaultAsync(cancellationToken).ConfigureAwait(false);
@@ -202,10 +209,8 @@ public override async IAsyncEnumerable<TRecord> GetAsync(
             throw new NotSupportedException(VectorDataStrings.IncludeVectorsNotSupportedWithEmbeddingGeneration);
         }
 
-        var stringKeys = keys is IEnumerable<string> k ? k : keys.Cast<string>();
-
         using var cursor = await this
-            .FindAsync(this.GetFilterByIds(stringKeys), top: null, skip: null, includeVectors, sortDefinition: null, cancellationToken)
+            .FindAsync(this.GetFilterByIds(keys), top: null, skip: null, includeVectors, sortDefinition: null, cancellationToken)
             .ConfigureAwait(false);
 
         while (await cursor.MoveNextAsync(cancellationToken).ConfigureAwait(false))
@@ -252,14 +257,17 @@ private async Task UpsertCoreAsync(TRecord record, int recordIndex, IReadOnlyLis
         var replaceOptions = new ReplaceOptions { IsUpsert = true };
         var storageModel = this._mapper.MapFromDataToStorageModel(record, recordIndex, generatedEmbeddings);
 
-        var key = storageModel[MongoConstants.MongoReservedKeyPropertyName].AsString;
+        var key = GetStorageKey(storageModel);
 
         await this.RunOperationAsync(OperationName, async () =>
             await this._mongoCollection
                 .ReplaceOneAsync(this.GetFilterById(key), storageModel, replaceOptions, cancellationToken)
                 .ConfigureAwait(false)).ConfigureAwait(false);
     }
 
+    private static TKey GetStorageKey(BsonDocument document)
+        => (TKey)BsonTypeMapper.MapToDotNetValue(document[MongoConstants.MongoReservedKeyPropertyName]);
+
     private static async ValueTask<(IEnumerable<TRecord> records, IReadOnlyList<Embedding>?[]?)> ProcessEmbeddingsAsync(
         CollectionModel model,
         IEnumerable<TRecord> records,
@@ -562,11 +570,40 @@ private async IAsyncEnumerable<VectorSearchResult<TRecord>> EnumerateAndMapSearc
         }
     }
 
-    private FilterDefinition<BsonDocument> GetFilterById(string id)
-        => Builders<BsonDocument>.Filter.Eq(document => document[MongoConstants.MongoReservedKeyPropertyName], id);
+    private FilterDefinition<BsonDocument> GetFilterById(TKey id)
+    {
+        // Use cached key serialization info but fall back to BsonValueFactory for dynamic mapper.
+        var bsonValue = this._keySerializationInfo?.SerializeValue(id) ?? BsonValueFactory.Create(id);
+        return Builders<BsonDocument>.Filter.Eq(MongoConstants.MongoReservedKeyPropertyName, bsonValue);
+    }
+
+    private FilterDefinition<BsonDocument> GetFilterByIds(IEnumerable<TKey> ids)
+    {
+        // Use cached key serialization info but fall back to BsonValueFactory for dynamic mapper.
+        var bsonValues = this._keySerializationInfo?.SerializeValues(ids) ?? (BsonArray)BsonValueFactory.Create(ids);
+        return Builders<BsonDocument>.Filter.In(MongoConstants.MongoReservedKeyPropertyName, bsonValues);
+    }
 
-    private FilterDefinition<BsonDocument> GetFilterByIds(IEnumerable<string> ids)
-        => Builders<BsonDocument>.Filter.In(document => document[MongoConstants.MongoReservedKeyPropertyName].AsString, ids);
+    private BsonSerializationInfo GetKeySerializationInfo()
+    {
+        var documentSerializer = BsonSerializer.LookupSerializer<TRecord>();
+        if (documentSerializer is null)
+        {
+            throw new InvalidOperationException($"BsonSerializer not found for type '{typeof(TRecord)}'");
+        }
+
+        if (documentSerializer is not IBsonDocumentSerializer bsonDocumentSerializer)
+        {
+            throw new InvalidOperationException($"BsonSerializer for type '{typeof(TRecord)}' does not implement IBsonDocumentSerializer");
+        }
+
+        if (!bsonDocumentSerializer.TryGetMemberSerializationInfo(this._model.KeyProperty.ModelName, out var keySerializationInfo))
+        {
+            throw new InvalidOperationException($"BsonSerializer for type '{typeof(TRecord)}' does not recognize key property {this._model.KeyProperty.ModelName}");
+        }
+
+        return keySerializationInfo;
+    }
 
     private async Task<bool> InternalCollectionExistsAsync(CancellationToken cancellationToken)
     {

dotnet/src/VectorData/CosmosMongoDB/CosmosMongoCollectionCreateMapping.cs

@@ -112,7 +112,7 @@ private static string GetIndexKind(string? indexKind, string vectorPropertyName)
         {
             IndexKind.Hnsw => "vector-hnsw",
             IndexKind.IvfFlat => "vector-ivf",
-            _ => throw new InvalidOperationException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
+            _ => throw new NotSupportedException($"Index kind '{indexKind}' on {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
         };
 
     /// <summary>
@@ -124,6 +124,6 @@ private static string GetDistanceFunction(string? distanceFunction, string vecto
             DistanceFunction.CosineDistance => "COS",
             DistanceFunction.DotProductSimilarity => "IP",
             DistanceFunction.EuclideanDistance => "L2",
-            _ => throw new InvalidOperationException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
+            _ => throw new NotSupportedException($"Distance function '{distanceFunction}' for {nameof(VectorStoreVectorProperty)} '{vectorPropertyName}' is not supported by the Azure CosmosDB for MongoDB VectorStore.")
         };
 }