Support dictionary responses (#6874) (#6879)

* Support dictionary responses

Includes improvements and fixes for Properties serialization

* Fix BOM

Co-authored-by: Steve Gordon <sgordon@hotmail.co.uk>

Author: github-actions[bot]

.github/workflows/integration-jobs.yml

@@ -26,7 +26,8 @@ jobs:
           '8.2.3',
           '8.3.3',
           '8.4.3',
-          '8.5.0-SNAPSHOT',
+          "8.5.0",
+          '8.6.0-SNAPSHOT',
           'latest-8'
         ]
 

build/scripts/Testing.fs

@@ -46,7 +46,7 @@ module Tests =
             sprintf "tests/%s.runsettings" prefix
 
         Directory.CreateDirectory Paths.BuildOutput |> ignore
-        let command = ["test"; proj; "--nologo"; "-c"; "Release"; "-s"; runSettings; "--no-build"]
+        let command = ["test"; proj; "--nologo"; "-c"; "Release"; "-s"; runSettings; "--no-build"; "--blame"]
 
         let wantsTrx =
             let wants = match args.CommandArguments with | Integration a -> a.TrxExport | Test t -> t.TrxExport | _ -> false

src/Elastic.Clients.Elasticsearch/Api/IndexManagement/MappingResponse.cs

@@ -0,0 +1,26 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System.Collections.Generic;
+using Elastic.Clients.Elasticsearch.Mapping;
+
+namespace Elastic.Clients.Elasticsearch.IndexManagement;
+
+public partial class MappingResponse
+{
+	public IReadOnlyDictionary<IndexName, IndexMappingRecord> Indices => BackingDictionary;
+}
+
+public static class GetMappingResponseExtensions
+{
+	public static TypeMapping GetMappingFor<T>(this MappingResponse response) => response.GetMappingFor(typeof(T));
+
+	public static TypeMapping GetMappingFor(this MappingResponse response, IndexName index)
+	{
+		if (index.IsNullOrEmpty())
+			return null;
+
+		return response.Indices.TryGetValue(index, out var indexMappings) ? indexMappings.Mappings : null;
+	}
+}

src/Elastic.Clients.Elasticsearch/Core/Exceptions/ThrowHelper.cs

@@ -0,0 +1,22 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Text.Json;
+
+namespace Elastic.Clients.Elasticsearch;
+
+internal static class ThrowHelper
+{
+	[MethodImpl(MethodImplOptions.NoInlining)]
+	internal static void ThrowJsonException(string? message = null) => throw new JsonException(message);
+
+	[MethodImpl(MethodImplOptions.NoInlining)]
+	internal static void ThrowUnknownTaggedUnionVariantJsonException(string variantTag, Type interfaceType) =>
+		throw new JsonException($"Encounted an unsupported variant tag '{variantTag}' on '{SimplifiedFullName(interfaceType)}', which could not be deserialised.");
+
+	[MethodImpl(MethodImplOptions.NoInlining)]
+	private static string SimplifiedFullName(Type type) => type.FullName.Substring(30);
+}

src/Elastic.Clients.Elasticsearch/Core/Infer/IndexName/IndexName.cs

@@ -123,12 +123,13 @@ private bool EqualsMarker(IndexName other)
 	{
 		if (other == null)
 			return false;
+
 		if (!Name.IsNullOrEmpty() && !other.Name.IsNullOrEmpty())
 			return EqualsString(PrefixClusterName(other, other.Name));
 
 		if ((!Cluster.IsNullOrEmpty() || !other.Cluster.IsNullOrEmpty()) && Cluster != other.Cluster)
 			return false;
 
-		return Type != null && other?.Type != null && Type == other.Type;
+		return Type is not null && other?.Type is not null && Type == other.Type;
 	}
 }

src/Elastic.Clients.Elasticsearch/Core/IsAReadOnlyDictionary.cs

@@ -19,9 +19,8 @@ internal IsAReadOnlyDictionary(IReadOnlyDictionary<TKey, TValue> backingDictiona
 			return;
 
 		var dictionary = new Dictionary<TKey, TValue>(backingDictionary.Count);
+
 		foreach (var key in backingDictionary.Keys)
-			// ReSharper disable once VirtualMemberCallInConstructor
-			// expect all implementations of Sanitize to be pure
 			dictionary[Sanitize(key)] = backingDictionary[key];
 
 		BackingDictionary = dictionary;

src/Elastic.Clients.Elasticsearch/Core/Response/DictionaryResponse.cs

@@ -0,0 +1,24 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using Elastic.Transport.Products.Elasticsearch;
+
+namespace Elastic.Clients.Elasticsearch;
+
+public abstract class DictionaryResponse<TKey, TValue> : ElasticsearchResponseBase
+{
+	internal DictionaryResponse(IReadOnlyDictionary<TKey, TValue> dictionary)
+	{
+		if (dictionary is null)
+			throw new ArgumentNullException(nameof(dictionary));
+
+		BackingDictionary = dictionary;
+	}
+
+	internal DictionaryResponse() => BackingDictionary = EmptyReadOnly<TKey, TValue>.Dictionary;
+
+	protected IReadOnlyDictionary<TKey, TValue> BackingDictionary { get; }
+}

src/Elastic.Clients.Elasticsearch/Core/Response/ResolvableDictionaryProxy.cs

@@ -0,0 +1,59 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using Elastic.Transport;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace Elastic.Clients.Elasticsearch;
+
+/// <summary>
+/// A proxy dictionary that is settings-aware to correctly handle IUrlParameter-based keys such as IndexName.
+/// </summary>
+public sealed class ResolvableDictionaryProxy<TKey, TValue> : IIsAReadOnlyDictionary<TKey, TValue>
+		where TKey : IUrlParameter
+{
+	private readonly IElasticsearchClientSettings _elasticsearchClientSettings;
+
+	internal ResolvableDictionaryProxy(IElasticsearchClientSettings elasticsearchClientSettings, IReadOnlyDictionary<TKey, TValue> backingDictionary)
+	{
+		_elasticsearchClientSettings = elasticsearchClientSettings;
+
+		if (backingDictionary == null)
+			return;
+
+		Original = backingDictionary;
+
+		var dictionary = new Dictionary<string, TValue>(backingDictionary.Count);
+
+		foreach (var key in backingDictionary.Keys)
+			dictionary[Sanitize(key)] = backingDictionary[key];
+
+		BackingDictionary = dictionary;
+	}
+
+	public int Count => BackingDictionary.Count;
+
+	public TValue this[TKey key] => BackingDictionary.TryGetValue(Sanitize(key), out var v) ? v : default;
+	public TValue this[string key] => BackingDictionary.TryGetValue(key, out var v) ? v : default;
+
+	public IEnumerable<TKey> Keys => Original.Keys;
+	public IEnumerable<string> ResolvedKeys => BackingDictionary.Keys;
+
+	public IEnumerable<TValue> Values => BackingDictionary.Values;
+	internal IReadOnlyDictionary<string, TValue> BackingDictionary { get; } = EmptyReadOnly<string, TValue>.Dictionary;
+	private IReadOnlyDictionary<TKey, TValue> Original { get; } = EmptyReadOnly<TKey, TValue>.Dictionary;
+
+	IEnumerator IEnumerable.GetEnumerator() => Original.GetEnumerator();
+
+	IEnumerator<KeyValuePair<TKey, TValue>> IEnumerable<KeyValuePair<TKey, TValue>>.GetEnumerator() =>
+		Original.GetEnumerator();
+
+	public bool ContainsKey(TKey key) => BackingDictionary.ContainsKey(Sanitize(key));
+
+	public bool TryGetValue(TKey key, out TValue value) =>
+		BackingDictionary.TryGetValue(Sanitize(key), out value);
+
+	private string Sanitize(TKey key) => key?.GetString(_elasticsearchClientSettings);
+}

src/Elastic.Clients.Elasticsearch/Serialization/DefaultRequestResponseSerializer.cs

@@ -2,7 +2,6 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using System;
 using System.IO;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -47,10 +46,11 @@ public DefaultRequestResponseSerializer(IElasticsearchClientSettings settings)
 					new SelfTwoWaySerializableConverterFactory(settings),
 					new IndicesJsonConverter(settings),
 					new IdsConverter(settings),
-					new IsADictionaryConverter(),
+					new IsADictionaryConverterFactory(),
 					new ResponseItemConverterFactory(),
 					new UnionConverter(),
-					new ExtraSerializationData(settings)
+					new ExtraSerializationData(settings),
+					new DictionaryResponseConverterFactory(settings)
 				},
 			PropertyNamingPolicy = JsonNamingPolicy.CamelCase
 		};

src/Elastic.Clients.Elasticsearch/Serialization/DictionaryResponseConverter.cs

@@ -0,0 +1,87 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Elastic.Transport;
+
+namespace Elastic.Clients.Elasticsearch.Serialization;
+
+internal sealed class DictionaryResponseConverterFactory : JsonConverterFactory
+{
+	private readonly IElasticsearchClientSettings _settings;
+
+	public DictionaryResponseConverterFactory(IElasticsearchClientSettings settings) => _settings = settings;
+
+	public override bool CanConvert(Type typeToConvert) =>
+		typeToConvert.BaseType is not null &&
+		typeToConvert.BaseType.IsGenericType &&
+		typeToConvert.BaseType.GetGenericTypeDefinition() == typeof(DictionaryResponse<,>);
+
+	public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
+	{
+		var args = typeToConvert.BaseType.GetGenericArguments();
+
+		var keyType = args[0];
+		var valueType = args[1];
+
+		if (keyType.IsClass)
+		{
+			if (keyType == typeof(IndexName))
+			{
+				return (JsonConverter)Activator.CreateInstance(
+					typeof(ResolvableDictionaryResponseConverterInner<,,>).MakeGenericType(typeToConvert, keyType, valueType), _settings);
+			}
+
+			return (JsonConverter)Activator.CreateInstance(
+				typeof(DictionaryResponseConverterInner<,,>).MakeGenericType(typeToConvert, keyType, valueType));
+		}
+
+		return null;
+	}
+
+	private class DictionaryResponseConverterInner<TType, TKey, TValue> : JsonConverter<TType>
+		where TKey : class
+		where TType : DictionaryResponse<TKey, TValue>, new()
+	{
+		public override TType? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+		{
+			var dictionary = JsonSerializer.Deserialize<Dictionary<TKey, TValue>>(ref reader, options);
+
+			if (dictionary is null)
+				return null;
+
+			return (TType)Activator.CreateInstance(typeof(TType), new object[] { dictionary });
+		}
+
+		public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options) =>
+			throw new NotImplementedException("Response converters do not support serialization.");
+	}
+
+	private class ResolvableDictionaryResponseConverterInner<TType, TKey, TValue> : JsonConverter<TType>
+		where TKey : class, IUrlParameter
+		where TType : DictionaryResponse<TKey, TValue>, new()
+	{
+		private readonly IElasticsearchClientSettings _settings;
+
+		public ResolvableDictionaryResponseConverterInner(IElasticsearchClientSettings settings) => _settings = settings;
+
+		public override TType? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+		{
+			var dictionary = JsonSerializer.Deserialize<Dictionary<TKey, TValue>>(ref reader, options);
+
+			if (dictionary is null)
+				return null;
+
+			var dictionaryProxy = new ResolvableDictionaryProxy<TKey, TValue>(_settings, dictionary);
+
+			return (TType)Activator.CreateInstance(typeof(TType), new object[] { dictionaryProxy });
+		}
+
+		public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOptions options) =>
+			throw new NotImplementedException("Response converters do not support serialization.");
+	}
+}

src/Elastic.Clients.Elasticsearch/Serialization/ThrowHelper.cs renamed to src/Elastic.Clients.Elasticsearch/Serialization/IUnionVerifiable.cs

@@ -2,13 +2,9 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using System.Runtime.CompilerServices;
-using System.Text.Json;
-
 namespace Elastic.Clients.Elasticsearch.Serialization;
 
-internal class ThrowHelper
+internal interface IUnionVerifiable
 {
-	[MethodImpl(MethodImplOptions.NoInlining)]
-	public static void ThrowJsonException(string? message = null) => throw new JsonException(message);
+	bool IsSuccessful { get; }
 }

src/Elastic.Clients.Elasticsearch/Serialization/IsADictionaryConverter.cs renamed to src/Elastic.Clients.Elasticsearch/Serialization/IsADictionaryConverterFactory.cs

@@ -6,14 +6,14 @@
 using System.Collections.Generic;
 using System.Text.Json;
 using System.Text.Json.Serialization;
+using Elastic.Clients.Elasticsearch.Mapping;
 
 namespace Elastic.Clients.Elasticsearch.Serialization;
 
-// TODO : We need to handle these cases https://github.com/elastic/elasticsearch-specification/pull/1589
-
-internal sealed class IsADictionaryConverter : JsonConverterFactory
+internal sealed class IsADictionaryConverterFactory : JsonConverterFactory
 {
 	public override bool CanConvert(Type typeToConvert) =>
+		typeToConvert.Name != nameof(Properties) && // Properties has it's own converter assigned
 		typeToConvert.BaseType is not null &&
 		typeToConvert.BaseType.IsGenericType &&
 		typeToConvert.BaseType.GetGenericTypeDefinition() == typeof(IsADictionary<,>);
@@ -52,8 +52,3 @@ public override void Write(Utf8JsonWriter writer, TType value, JsonSerializerOpt
 			JsonSerializer.Serialize<Dictionary<TKey, TValue>>(writer, value.BackingDictionary, options);
 	}
 }
-
-internal interface IUnionVerifiable
-{
-	bool IsSuccessful { get; }
-}