diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index 733b885b39a..e7b535e6995 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -184,6 +184,47 @@ static async Task ToChatResponseAsync( } } + /// + /// Coalesces image result content elements in the provided list of items. + /// Unlike other content coalescing methods, this will coalesce non-sequential items based on their Name property, + /// and it will replace earlier items with later ones when duplicates are found. + /// + private static void CoalesceImageResultContent(IList contents) + { + Dictionary? imageResultIndexById = null; + bool hasRemovals = false; + + for (int i = 0; i < contents.Count; i++) + { + if (contents[i] is ImageGenerationToolResultContent imageResult && !string.IsNullOrEmpty(imageResult.ImageId)) + { + // Check if there's an existing ImageGenerationToolResultContent with the same ImageId to replace + if (imageResultIndexById is null) + { + imageResultIndexById = new(StringComparer.Ordinal); + } + + if (imageResultIndexById.TryGetValue(imageResult.ImageId!, out int existingIndex)) + { + // Replace the existing imageResult with the new one + contents[existingIndex] = imageResult; + contents[i] = null!; // Mark the current one for removal, then remove in single o(n) pass + hasRemovals = true; + } + else + { + imageResultIndexById[imageResult.ImageId!] = i; + } + } + } + + // Remove all of the null slots left over from the coalescing process. + if (hasRemovals) + { + RemoveNullContents(contents); + } + } + /// Coalesces sequential content elements. internal static void CoalesceContent(IList contents) { @@ -219,6 +260,8 @@ internal static void CoalesceContent(IList contents) return content; }); + CoalesceImageResultContent(contents); + Coalesce( contents, mergeSingle: false, @@ -394,29 +437,35 @@ static bool TryAsCoalescable(AIContent content, [NotNullWhen(true)] out TContent } // Remove all of the null slots left over from the coalescing process. - if (contents is List contentsList) - { - _ = contentsList.RemoveAll(u => u is null); - } - else - { - int nextSlot = 0; - int contentsCount = contents.Count; - for (int i = 0; i < contentsCount; i++) - { - if (contents[i] is { } content) - { - contents[nextSlot++] = content; - } - } + RemoveNullContents(contents); + } + } - for (int i = contentsCount - 1; i >= nextSlot; i--) + private static void RemoveNullContents(IList contents) + where T : class + { + if (contents is List contentsList) + { + _ = contentsList.RemoveAll(u => u is null); + } + else + { + int nextSlot = 0; + int contentsCount = contents.Count; + for (int i = 0; i < contentsCount; i++) + { + if (contents[i] is { } content) { - contents.RemoveAt(i); + contents[nextSlot++] = content; } + } - Debug.Assert(nextSlot == contents.Count, "Expected final count to equal list length."); + for (int i = contentsCount - 1; i >= nextSlot; i--) + { + contents.RemoveAt(i); } + + Debug.Assert(nextSlot == contents.Count, "Expected final count to equal list length."); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs index c4a9f8ba97c..f1ad70cd22f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseUpdate.cs @@ -20,7 +20,7 @@ namespace Microsoft.Extensions.AI; /// /// /// The relationship between and is -/// codified in the and +/// codified in the and /// , which enable bidirectional conversions /// between the two. Note, however, that the provided conversions might be lossy, for example, if multiple /// updates all have different objects whereas there's only one slot for @@ -58,6 +58,29 @@ public ChatResponseUpdate(ChatRole? role, IList? contents) _contents = contents; } + /// + /// Creates a new ChatResponseUpdate instance that is a copy of the current object. + /// + /// The cloned object is a shallow copy; reference-type properties will reference the same + /// objects as the original. Use this method to duplicate the response update for further modification without + /// affecting the original instance. + /// A new ChatResponseUpdate object with the same property values as the current instance. + public ChatResponseUpdate Clone() => + new() + { + AdditionalProperties = AdditionalProperties, + AuthorName = AuthorName, + Contents = Contents, + CreatedAt = CreatedAt, + ConversationId = ConversationId, + FinishReason = FinishReason, + MessageId = MessageId, + ModelId = ModelId, + RawRepresentation = RawRepresentation, + ResponseId = ResponseId, + Role = Role, + }; + /// Gets or sets the name of the author of the response update. public string? AuthorName { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolCallContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolCallContent.cs new file mode 100644 index 00000000000..f5703a39e69 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolCallContent.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents the invocation of an image generation tool call by a hosted service. +/// +[Experimental("MEAI001")] +public sealed class ImageGenerationToolCallContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + public ImageGenerationToolCallContent() + { + } + + /// + /// Gets or sets the unique identifier of the image generation item. + /// + public string? ImageId { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolResultContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolResultContent.cs new file mode 100644 index 00000000000..2ce6d5045f7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageGenerationToolResultContent.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents an image generation tool call invocation by a hosted service. +/// +/// +/// This content type represents when a hosted AI service invokes an image generation tool. +/// It is informational only and represents the call itself, not the result. +/// +[Experimental("MEAI001")] +public sealed class ImageGenerationToolResultContent : AIContent +{ + /// + /// Initializes a new instance of the class. + /// + public ImageGenerationToolResultContent() + { + } + + /// + /// Gets or sets the unique identifier of the image generation item. + /// + public string? ImageId { get; set; } + + /// + /// Gets or sets the generated content items. + /// + /// + /// Content is typically for images streamed from the tool, or for remotely hosted images, but + /// can also be provider-specific content types that represent the generated images. + /// + public IList? Outputs { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationOptions.cs index fce02e9e88e..586fbcc8bbe 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Image/ImageGenerationOptions.cs @@ -81,6 +81,11 @@ protected ImageGenerationOptions(ImageGenerationOptions? other) /// public ImageGenerationResponseFormat? ResponseFormat { get; set; } + /// + /// Gets or sets the number of intermediate streaming images to generate. + /// + public int? StreamingCount { get; set; } + /// Gets or sets any additional properties associated with the options. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 64d767a4ea1..b5ef3774b45 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -1247,6 +1247,10 @@ "Member": "Microsoft.Extensions.AI.ChatResponseUpdate.ChatResponseUpdate(Microsoft.Extensions.AI.ChatRole? role, System.Collections.Generic.IList? contents);", "Stage": "Stable" }, + { + "Member": "Microsoft.Extensions.AI.ChatResponseUpdate Microsoft.Extensions.AI.ChatResponseUpdate.Clone();", + "Stage": "Stable" + }, { "Member": "override string Microsoft.Extensions.AI.ChatResponseUpdate.ToString();", "Stage": "Stable" diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs new file mode 100644 index 00000000000..aca072653ab --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Tools/HostedImageGenerationTool.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.AI; + +/// Represents a hosted tool that can be specified to an AI service to enable it to perform image generation. +/// +/// This tool does not itself implement image generation. It is a marker that can be used to inform a service +/// that the service is allowed to perform image generation if the service is capable of doing so. +/// +[Experimental("MEAI001")] +public class HostedImageGenerationTool : AITool +{ + /// + /// Initializes a new instance of the class with the specified options. + /// + public HostedImageGenerationTool() + { + } + + /// + /// Gets or sets the options used to configure image generation. + /// + public ImageGenerationOptions? Options { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index 846b7de1e9b..6913e999936 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -156,7 +156,7 @@ internal static ChatResponse FromOpenAIResponse(OpenAIResponse openAIResponse, R if (openAIResponse.OutputItems is not null) { - response.Messages = [.. ToChatMessages(openAIResponse.OutputItems)]; + response.Messages = [.. ToChatMessages(openAIResponse.OutputItems, openAIOptions)]; if (response.Messages.LastOrDefault() is { } lastMessage && openAIResponse.Error is { } error) { @@ -172,7 +172,7 @@ internal static ChatResponse FromOpenAIResponse(OpenAIResponse openAIResponse, R return response; } - internal static IEnumerable ToChatMessages(IEnumerable items) + internal static IEnumerable ToChatMessages(IEnumerable items, ResponseCreationOptions? options = null) { ChatMessage? message = null; @@ -236,6 +236,10 @@ internal static IEnumerable ToChatMessages(IEnumerable().Select(c => c.VectorStoreId) ?? [], fileSearchTool.MaximumResultCount); + case HostedImageGenerationTool imageGenerationTool: + return ToImageResponseTool(imageGenerationTool); + case HostedCodeInterpreterTool codeTool: return ResponseTool.CreateCodeInterpreterTool( new CodeInterpreterToolContainer(codeTool.Inputs?.OfType().Select(f => f.FileId).ToList() is { Count: > 0 } ids ? @@ -584,6 +604,40 @@ internal static FunctionTool ToResponseTool(AIFunctionDeclaration aiFunction, Ch aiFunction.Description); } + internal static ImageGenerationTool ToImageResponseTool(HostedImageGenerationTool imageGenerationTool) + { + ImageGenerationTool result = new(); + ImageGenerationOptions? imageGenerationOptions = imageGenerationTool.Options; + + // Model: Image generation model + result.Model = imageGenerationOptions?.ModelId; + + // Size: Image dimensions (e.g., 1024x1024, 1024x1536) + if (imageGenerationOptions?.ImageSize is not null) + { + result.Size = new ImageGenerationToolSize( + imageGenerationOptions.ImageSize.Value.Width, + imageGenerationOptions.ImageSize.Value.Height); + } + + // OutputFileFormat: File output format + if (imageGenerationOptions?.MediaType is not null) + { + result.OutputFileFormat = imageGenerationOptions.MediaType switch + { + "image/png" => ImageGenerationToolOutputFileFormat.Png, + "image/jpeg" => ImageGenerationToolOutputFileFormat.Jpeg, + "image/webp" => ImageGenerationToolOutputFileFormat.Webp, + _ => null, + }; + } + + // PartialImageCount: Whether to return partial images during generation + result.PartialImageCount ??= imageGenerationOptions?.StreamingCount; + + return result; + } + /// Creates a from a . private static ChatRole ToChatRole(MessageRole? role) => role switch @@ -1220,6 +1274,64 @@ private static void AddCodeInterpreterContents(CodeInterpreterCallResponseItem c }); } + private static void AddImageGenerationContents(ImageGenerationCallResponseItem outputItem, ResponseCreationOptions? options, IList contents) + { + var imageGenTool = options?.Tools.OfType().FirstOrDefault(); + string outputFormat = imageGenTool?.OutputFileFormat?.ToString() ?? "png"; + + contents.Add(new ImageGenerationToolCallContent + { + ImageId = outputItem.Id, + }); + + contents.Add(new ImageGenerationToolResultContent + { + ImageId = outputItem.Id, + RawRepresentation = outputItem, + Outputs = new List + { + new DataContent(outputItem.ImageResultBytes, $"image/{outputFormat}") + } + }); + } + + private static ImageGenerationToolResultContent GetImageGenerationResult(StreamingResponseImageGenerationCallPartialImageUpdate update, ResponseCreationOptions? options) + { + var imageGenTool = options?.Tools.OfType().FirstOrDefault(); + var outputType = imageGenTool?.OutputFileFormat?.ToString() ?? "png"; + + var bytes = update.PartialImageBytes; + + if (bytes is null || bytes.Length == 0) + { + // workaround https://github.com/openai/openai-dotnet/issues/809 + if (update.Patch.TryGetJson("$.partial_image_b64"u8, out var jsonBytes)) + { + Utf8JsonReader reader = new(jsonBytes.Span); + _ = reader.Read(); + bytes = BinaryData.FromBytes(reader.GetBytesFromBase64()); + } + } + + return new ImageGenerationToolResultContent + { + ImageId = update.ItemId, + RawRepresentation = update, + Outputs = new List + { + new DataContent(bytes, $"image/{outputType}") + { + AdditionalProperties = new() + { + [nameof(update.ItemId)] = update.ItemId, + [nameof(update.OutputIndex)] = update.OutputIndex, + [nameof(update.PartialImageIndex)] = update.PartialImageIndex + } + } + } + }; + } + private static OpenAIResponsesContinuationToken? CreateContinuationToken(OpenAIResponse openAIResponse) { return CreateContinuationToken( diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs new file mode 100644 index 00000000000..436adeb2295 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClient.cs @@ -0,0 +1,518 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A delegating chat client that enables image generation capabilities by converting instances to function tools. +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// employed is also thread-safe for concurrent use. +/// +/// +/// This client automatically detects instances in the collection +/// and replaces them with equivalent function tools that the chat client can invoke to perform image generation and editing operations. +/// +/// +[Experimental("MEAI001")] +public sealed class ImageGeneratingChatClient : DelegatingChatClient +{ + /// + /// Specifies how image and other data content is handled when passing data to an inner client. + /// + /// + /// Use this enumeration to control whether images in the data content are passed as-is, replaced + /// with unique identifiers, or only generated images are replaced. This setting affects how downstream clients + /// receive and process image data. + /// Reducing what's passed downstream can help manage the context window. + /// + public enum DataContentHandling + { + /// Pass all DataContent to inner client. + None, + + /// Replace all images with unique identifiers when passing to inner client. + AllImages, + + /// Replace only images that were produced by past image generation requests with unique identifiers when passing to inner client. + GeneratedImages + } + + private const string ImageKey = "meai_image"; + + private readonly IImageGenerator _imageGenerator; + private readonly DataContentHandling _dataContentHandling; + + /// Initializes a new instance of the class. + /// The underlying . + /// An instance that will be used for image generation operations. + /// Specifies how to handle instances when passing messages to the inner client. + /// The default is . + /// or is . + public ImageGeneratingChatClient(IChatClient innerClient, IImageGenerator imageGenerator, DataContentHandling dataContentHandling = DataContentHandling.AllImages) + : base(innerClient) + { + _imageGenerator = Throw.IfNull(imageGenerator); + _dataContentHandling = dataContentHandling; + } + + /// + public override async Task GetResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + var requestState = new RequestState(_imageGenerator, _dataContentHandling); + + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = requestState.ProcessChatOptions(options); + var processedMessages = requestState.ProcessChatMessages(messages); + + // Get response from base implementation + var response = await base.GetResponseAsync(processedMessages, processedOptions, cancellationToken); + + // Replace FunctionResultContent instances with generated image content + foreach (var message in response.Messages) + { + message.Contents = requestState.ReplaceImageGenerationFunctionResults(message.Contents); + } + + return response; + } + + /// + public override async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + + var requestState = new RequestState(_imageGenerator, _dataContentHandling); + + // Process the chat options to replace HostedImageGenerationTool with functions + var processedOptions = requestState.ProcessChatOptions(options); + var processedMessages = requestState.ProcessChatMessages(messages); + + await foreach (var update in base.GetStreamingResponseAsync(processedMessages, processedOptions, cancellationToken)) + { + // Replace any FunctionResultContent instances with generated image content + var newContents = requestState.ReplaceImageGenerationFunctionResults(update.Contents); + + if (!ReferenceEquals(newContents, update.Contents)) + { + // Create a new update instance with modified contents + var modifiedUpdate = update.Clone(); + modifiedUpdate.Contents = newContents; + yield return modifiedUpdate; + } + else + { + yield return update; + } + } + } + + /// Provides a mechanism for releasing unmanaged resources. + /// to dispose managed resources; otherwise, . + protected override void Dispose(bool disposing) + { + if (disposing) + { + _imageGenerator.Dispose(); + } + + base.Dispose(disposing); + } + + /// + /// Contains all the per-request state and methods for handling image generation requests. + /// This class is created fresh for each request to ensure thread safety. + /// This class is not exposed publicly and does not own any of it's resources. + /// + private sealed class RequestState + { + private readonly IImageGenerator _imageGenerator; + private readonly DataContentHandling _dataContentHandling; + private readonly HashSet _toolNames = new(StringComparer.Ordinal); + private readonly Dictionary> _imageContentByCallId = []; + private readonly Dictionary _imageContentById = new(StringComparer.OrdinalIgnoreCase); + private ImageGenerationOptions? _imageGenerationOptions; + + public RequestState(IImageGenerator imageGenerator, DataContentHandling dataContentHandling) + { + _imageGenerator = imageGenerator; + _dataContentHandling = dataContentHandling; + } + + /// + /// Processes the chat messages to replace images in data content with unique identifiers as needed. + /// All images will be stored for later retrieval during image editing operations. + /// See for details on image replacement behavior. + /// + /// Messages to process. + /// Processed messages, or the original messages if no changes were made. + public IEnumerable ProcessChatMessages(IEnumerable messages) + { + List? newMessages = null; + int messageIndex = 0; + foreach (var message in messages) + { + List? newContents = null; + for (int contentIndex = 0; contentIndex < message.Contents.Count; contentIndex++) + { + var content = message.Contents[contentIndex]; + + void ReplaceImage(string imageId, DataContent dataContent) + { + // Replace image with a placeholder text content, to give an indication to the model of its placement in the context + newContents ??= CopyList(message.Contents, contentIndex); + newContents.Add(new TextContent($"[{ImageKey}:{imageId}] available for edit.") + { + Annotations = dataContent.Annotations, + AdditionalProperties = dataContent.AdditionalProperties + }); + } + + if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) + { + // Store the image to make available for edit + var imageId = StoreImage(dataContent); + + if (_dataContentHandling == DataContentHandling.AllImages) + { + ReplaceImage(imageId, dataContent); + continue; // Skip adding the original content + } + } + else if (content is ImageGenerationToolResultContent toolResultContent) + { + foreach (var output in toolResultContent.Outputs ?? []) + { + if (output is DataContent generatedDataContent && generatedDataContent.HasTopLevelMediaType("image")) + { + // Store the image to make available for edit + var imageId = StoreImage(generatedDataContent, isGenerated: true); + + if (_dataContentHandling == DataContentHandling.AllImages || + _dataContentHandling == DataContentHandling.GeneratedImages) + { + ReplaceImage(imageId, generatedDataContent); + } + } + } + + if (_dataContentHandling == DataContentHandling.AllImages || + _dataContentHandling == DataContentHandling.GeneratedImages) + { + // skip adding the generated content + continue; + } + } + + // Add the original content if no replacement was made + newContents?.Add(content); + } + + if (newContents != null) + { + newMessages ??= [.. messages.Take(messageIndex)]; + var newMessage = message.Clone(); + newMessage.Contents = newContents; + newMessages.Add(newMessage); + } + else + { + newMessages?.Add(message); + + } + + messageIndex++; + } + + return newMessages ?? messages; + } + + public ChatOptions? ProcessChatOptions(ChatOptions? options) + { + if (options?.Tools is null || options.Tools.Count == 0) + { + return options; + } + + List? newTools = null; + var tools = options.Tools; + for (int i = 0; i < tools.Count; i++) + { + var tool = tools[i]; + + // remove all instances of HostedImageGenerationTool and store the options from the last one + if (tool is HostedImageGenerationTool imageGenerationTool) + { + _imageGenerationOptions = imageGenerationTool.Options; + + // for the first image generation tool, clone the options and insert our function tools + // remove any subsequent image generation tools + newTools ??= InitializeTools(tools, i); + } + else + { + newTools?.Add(tool); + } + } + + if (newTools is not null) + { + var newOptions = options.Clone(); + newOptions.Tools = newTools; + return newOptions; + } + + return options; + + List InitializeTools(IList existingTools, int toOffsetExclusive) + { +#if NET + ReadOnlySpan tools = +#else + AITool[] tools = +#endif + [ + AIFunctionFactory.Create(GenerateImageAsync), + AIFunctionFactory.Create(EditImageAsync), + AIFunctionFactory.Create(GetImagesForEdit) + ]; + + foreach (var tool in tools) + { + _toolNames.Add(tool.Name); + } + + var result = CopyList(existingTools, toOffsetExclusive, tools.Length); + result.AddRange(tools); + return result; + } + } + + /// + /// Replaces FunctionResultContent instances for image generation functions with actual generated image content. + /// We will have two messages + /// 1. Role: Assistant, FunctionCall + /// 2. Role: Tool, FunctionResult + /// We need to replace content from both but we shouldn't remove the messages. + /// If we do not then ChatClient's may not accept our altered history. + /// + /// When interating with a HostedImageGenerationTool we will have typically only see a single Message with + /// Role: Assistant that contains the DataContent (or a provider specific content, that's exposed as DataContent). + /// + /// The list of AI content to process. + public IList ReplaceImageGenerationFunctionResults(IList contents) + { + List? newContents = null; + + // Replace FunctionResultContent instances with generated image content + for (int i = contents.Count - 1; i >= 0; i--) + { + var content = contents[i]; + + // We must lookup by name because in the streaming case we have not yet been called to record the CallId. + if (content is FunctionCallContent functionCall && + _toolNames.Contains(functionCall.Name)) + { + // create a new list and omit the FunctionCallContent + newContents ??= CopyList(contents, i); + + if (functionCall.Name != nameof(GetImagesForEdit)) + { + newContents.Add(new ImageGenerationToolCallContent + { + ImageId = functionCall.CallId, + }); + } + } + else if (content is FunctionResultContent functionResult && + _imageContentByCallId.TryGetValue(functionResult.CallId, out var imageContents)) + { + newContents ??= CopyList(contents, i); + + if (imageContents.Any()) + { + // Insert ImageGenerationToolResultContent in its place, do not preserve the FunctionResultContent + newContents.Add(new ImageGenerationToolResultContent + { + ImageId = functionResult.CallId, + Outputs = imageContents + }); + } + + // Remove the mapping as it's no longer needed + _ = _imageContentByCallId.Remove(functionResult.CallId); + } + else + { + // keep the existing content if we have a new list + newContents?.Add(content); + } + } + + return newContents ?? contents; + } + + [Description("Generates images based on a text description.")] + public async Task GenerateImageAsync( + [Description("A detailed description of the image to generate")] string prompt, + CancellationToken cancellationToken = default) + { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return "No call ID available for image generation."; + } + + var request = new ImageGenerationRequest(prompt); + var options = _imageGenerationOptions ?? new ImageGenerationOptions(); + options.Count ??= 1; + + var response = await _imageGenerator.GenerateAsync(request, options, cancellationToken); + + if (response.Contents.Count == 0) + { + return "No image was generated."; + } + + List imageIds = []; + List imageContents = _imageContentByCallId[callId] = []; + foreach (var content in response.Contents) + { + if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + imageContents.Add(imageContent); + imageIds.Add(StoreImage(imageContent, true)); + } + } + + return "Generated image successfully."; + } + + [Description("Lists the identifiers of all images available for edit.")] + public IEnumerable GetImagesForEdit() + { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return ["No call ID available for image editing."]; + } + + _imageContentByCallId[callId] = []; + + return _imageContentById.Keys.AsEnumerable(); + } + + [Description("Edits an existing image based on a text description.")] + public async Task EditImageAsync( + [Description("A detailed description of the image to generate")] string prompt, + [Description($"The image to edit from one of the available image identifiers returned by {nameof(GetImagesForEdit)}")] string imageId, + CancellationToken cancellationToken = default) + { + // Get the call ID from the current function invocation context + var callId = FunctionInvokingChatClient.CurrentContext?.CallContent.CallId; + if (callId == null) + { + return "No call ID available for image editing."; + } + + if (string.IsNullOrEmpty(imageId)) + { + return "No imageId provided"; + } + + try + { + var originalImage = RetrieveImageContent(imageId); + if (originalImage == null) + { + return $"No image found with: {imageId}"; + } + + var request = new ImageGenerationRequest(prompt, [originalImage]); + var response = await _imageGenerator.GenerateAsync(request, _imageGenerationOptions, cancellationToken); + + if (response.Contents.Count == 0) + { + return "No edited image was generated."; + } + + List imageIds = []; + List imageContents = _imageContentByCallId[callId] = []; + foreach (var content in response.Contents) + { + if (content is DataContent imageContent && imageContent.MediaType.StartsWith("image/", StringComparison.OrdinalIgnoreCase)) + { + imageContents.Add(imageContent); + imageIds.Add(StoreImage(imageContent, true)); + } + } + + return "Edited image successfully."; + } + catch (FormatException) + { + return "Invalid image data format. Please provide a valid base64-encoded image."; + } + } + + private static List CopyList(IList original, int toOffsetExclusive, int additionalCapacity = 0) + { + var newList = new List(original.Count + additionalCapacity); + + // Copy all items up to and excluding the current index + for (int j = 0; j < toOffsetExclusive; j++) + { + newList.Add(original[j]); + } + + return newList; + } + + private DataContent? RetrieveImageContent(string imageId) + { + if (_imageContentById.TryGetValue(imageId, out var imageContent)) + { + return imageContent as DataContent; + } + + return null; + } + + private string StoreImage(DataContent imageContent, bool isGenerated = false) + { + // Generate a unique ID for the image if it doesn't have one + string? imageId = null; + if (imageContent.AdditionalProperties?.TryGetValue(ImageKey, out imageId) is false || imageId is null) + { + imageId = imageContent.Name ?? Guid.NewGuid().ToString(); + } + + if (isGenerated) + { + imageContent.AdditionalProperties ??= []; + imageContent.AdditionalProperties[ImageKey] = imageId; + } + + // Store the image content for later retrieval + _imageContentById[imageId] = imageContent; + + return imageId; + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs new file mode 100644 index 00000000000..241c851fd4e --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ImageGeneratingChatClientBuilderExtensions.cs @@ -0,0 +1,46 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental("MEAI001")] +public static class ImageGeneratingChatClientBuilderExtensions +{ + /// Adds image generation capabilities to the chat client pipeline. + /// The . + /// + /// An optional used for image generation operations. + /// If not supplied, a required instance will be resolved from the service provider. + /// + /// An optional callback that can be used to configure the instance. + /// The . + /// is . + /// + /// + /// This method enables the chat client to handle instances by converting them + /// into function tools that can be invoked by the underlying chat model to perform image generation and editing operations. + /// + /// + public static ChatClientBuilder UseImageGeneration( + this ChatClientBuilder builder, + IImageGenerator? imageGenerator = null, + Action? configure = null) + { + _ = Throw.IfNull(builder); + + return builder.Use((innerClient, services) => + { + imageGenerator ??= services.GetRequiredService(); + + var chatClient = new ImageGeneratingChatClient(innerClient, imageGenerator); + configure?.Invoke(chatClient); + return chatClient; + }); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs index 368e11b44d4..09f3600e522 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs @@ -9,6 +9,7 @@ using Xunit; #pragma warning disable SA1204 // Static elements should appear before instance elements +#pragma warning disable MEAI0001 // Suppress experimental warnings for testing namespace Microsoft.Extensions.AI; @@ -450,7 +451,7 @@ public async Task ToChatResponse_UpdatesProduceMultipleResponseMessages(bool use { ChatResponseUpdate[] updates = [ - + // First message - ID "msg1", AuthorName "Assistant" new(null, "Hi! ") { CreatedAt = new DateTimeOffset(2023, 1, 1, 10, 0, 0, TimeSpan.Zero), AuthorName = "Assistant" }, new(ChatRole.Assistant, "Hello") { MessageId = "msg1", CreatedAt = new DateTimeOffset(2024, 1, 1, 10, 0, 0, TimeSpan.Zero), AuthorName = "Assistant" }, @@ -822,6 +823,99 @@ await YieldAsync(updates).ToChatResponseAsync() : Assert.Equal(expected, response.CreatedAt); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ToChatResponse_CoalescesImageGenerationToolResultContent(bool useAsync) + { + // Create test image content with actual byte arrays + var image1 = new DataContent((byte[])[1, 2, 3, 4], "image/png") { Name = "image1.png" }; + var image2 = new DataContent((byte[])[5, 6, 7, 8], "image/jpeg") { Name = "image2.jpg" }; + var image3 = new DataContent((byte[])[9, 10, 11, 12], "image/png") { Name = "image3.png" }; + var image4 = new DataContent((byte[])[13, 14, 15, 16], "image/gif") { Name = "image4.gif" }; + + ChatResponseUpdate[] updates = + { + new(null, "Let's generate"), + new(null, " some images"), + + // Initial ImageGenerationToolResultContent with ID "img1" + new() { Contents = [new ImageGenerationToolResultContent { ImageId = "img1", Outputs = [image1] }] }, + + // Another ImageGenerationToolResultContent with different ID "img2" + new() { Contents = [new ImageGenerationToolResultContent { ImageId = "img2", Outputs = [image2] }] }, + + // Another ImageGenerationToolResultContent with same ID "img1" - should replace the first one + new() { Contents = [new ImageGenerationToolResultContent { ImageId = "img1", Outputs = [image3] }] }, + + // ImageGenerationToolResultContent with same ID "img2" - should replace the second one + new() { Contents = [new ImageGenerationToolResultContent { ImageId = "img2", Outputs = [image4] }] }, + + // Final text + new(null, "Here are those generated images"), + }; + + ChatResponse response = useAsync ? await YieldAsync(updates).ToChatResponseAsync() : updates.ToChatResponse(); + ChatMessage message = Assert.Single(response.Messages); + + // Should have 4 content items: 1 text (coalesced) + 2 image results (coalesced) + 1 text + Assert.Equal(4, message.Contents.Count); + + // Verify text content was coalesced properly + Assert.Equal("Let's generate some images", + Assert.IsType(message.Contents[0]).Text); + + // Get the image result contents + var imageResults = message.Contents.OfType().ToArray(); + Assert.Equal(2, imageResults.Length); + + // Verify the first image result (ID "img1") has the latest content (image3) + var firstImageResult = imageResults.First(ir => ir.ImageId == "img1"); + Assert.NotNull(firstImageResult.Outputs); + var firstOutput = Assert.Single(firstImageResult.Outputs); + Assert.Same(image3, firstOutput); // Should be the later image, not image1 + + // Verify the second image result (ID "img2") has the latest content (image4) + var secondImageResult = imageResults.First(ir => ir.ImageId == "img2"); + Assert.NotNull(secondImageResult.Outputs); + var secondOutput = Assert.Single(secondImageResult.Outputs); + Assert.Same(image4, secondOutput); // Should be the later image, not image2 + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ToChatResponse_ImageGenerationToolResultContentWithNullOrEmptyImageId_DoesNotCoalesce(bool useAsync) + { + var image1 = new DataContent((byte[])[1, 2, 3, 4], "image/png") { Name = "image1.png" }; + var image2 = new DataContent((byte[])[5, 6, 7, 8], "image/jpeg") { Name = "image2.jpg" }; + var image3 = new DataContent((byte[])[9, 10, 11, 12], "image/png") { Name = "image3.png" }; + + ChatResponseUpdate[] updates = + { + // ImageGenerationToolResultContent with null ImageId - should not coalesce + new() { Contents = [new ImageGenerationToolResultContent { ImageId = null, Outputs = [image1] }] }, + + // ImageGenerationToolResultContent with empty ImageId - should not coalesce + new() { Contents = [new ImageGenerationToolResultContent { ImageId = "", Outputs = [image2] }] }, + + // Another with null ImageId - should not coalesce with the first + new() { Contents = [new ImageGenerationToolResultContent { ImageId = null, Outputs = [image3] }] }, + }; + + ChatResponse response = useAsync ? await YieldAsync(updates).ToChatResponseAsync() : updates.ToChatResponse(); + ChatMessage message = Assert.Single(response.Messages); + + // Should have all 3 image result contents since they can't be coalesced + var imageResults = message.Contents.OfType().ToArray(); + Assert.Equal(3, imageResults.Length); + + // Verify each has its original content + Assert.Same(image1, imageResults[0].Outputs![0]); + Assert.Same(image2, imageResults[1].Outputs![0]); + Assert.Same(image3, imageResults[2].Outputs![0]); + } + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) { foreach (ChatResponseUpdate update in updates) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs index 413215d9a44..9727a58ac47 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateTests.cs @@ -167,4 +167,165 @@ public void JsonSerialization_Roundtrips() Assert.IsType(value); Assert.Equal("value", ((JsonElement)value!).GetString()); } + + [Fact] + public void Clone_CreatesShallowCopy() + { + // Arrange + var originalAdditionalProperties = new AdditionalPropertiesDictionary { ["key"] = "value" }; + var originalContents = new List { new TextContent("text1"), new TextContent("text2") }; + var originalRawRepresentation = new object(); + var originalCreatedAt = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero); + + var original = new ChatResponseUpdate + { + AdditionalProperties = originalAdditionalProperties, + AuthorName = "author", + Contents = originalContents, + CreatedAt = originalCreatedAt, + ConversationId = "conv123", + FinishReason = ChatFinishReason.ContentFilter, + MessageId = "msg456", + ModelId = "model789", + RawRepresentation = originalRawRepresentation, + ResponseId = "resp012", + Role = ChatRole.Assistant, + }; + + // Act + var clone = original.Clone(); + + // Assert - Different instances + Assert.NotSame(original, clone); + + // Assert - All properties copied correctly + Assert.Equal(original.AuthorName, clone.AuthorName); + Assert.Equal(original.Role, clone.Role); + Assert.Equal(original.CreatedAt, clone.CreatedAt); + Assert.Equal(original.ConversationId, clone.ConversationId); + Assert.Equal(original.FinishReason, clone.FinishReason); + Assert.Equal(original.MessageId, clone.MessageId); + Assert.Equal(original.ModelId, clone.ModelId); + Assert.Equal(original.ResponseId, clone.ResponseId); + + // Assert - Reference properties are shallow copied (same references) + Assert.Same(original.AdditionalProperties, clone.AdditionalProperties); + Assert.Same(original.Contents, clone.Contents); + Assert.Same(original.RawRepresentation, clone.RawRepresentation); + } + + [Fact] + public void Clone_WithNullProperties_CopiesCorrectly() + { + // Arrange + var original = new ChatResponseUpdate + { + Role = ChatRole.User, + ResponseId = "resp123" + }; + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Equal(ChatRole.User, clone.Role); + Assert.Equal("resp123", clone.ResponseId); + Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.AuthorName); + Assert.Null(clone.CreatedAt); + Assert.Null(clone.ConversationId); + Assert.Null(clone.FinishReason); + Assert.Null(clone.MessageId); + Assert.Null(clone.ModelId); + Assert.Null(clone.RawRepresentation); + Assert.Empty(clone.Contents); // Contents property initializes to empty list + } + + [Fact] + public void Clone_WithDefaultConstructor_CopiesCorrectly() + { + // Arrange + var original = new ChatResponseUpdate(); + + // Act + var clone = original.Clone(); + + // Assert + Assert.NotSame(original, clone); + Assert.Null(clone.AuthorName); + Assert.Null(clone.Role); + Assert.Empty(clone.Contents); + Assert.Null(clone.RawRepresentation); + Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.ResponseId); + Assert.Null(clone.MessageId); + Assert.Null(clone.CreatedAt); + Assert.Null(clone.FinishReason); + Assert.Null(clone.ConversationId); + Assert.Null(clone.ModelId); + } + + [Fact] + public void Clone_ModifyingClone_DoesNotAffectOriginal() + { + // Arrange + var original = new ChatResponseUpdate + { + AuthorName = "original_author", + Role = ChatRole.User, + ResponseId = "original_id", + ModelId = "original_model" + }; + + // Act + var clone = original.Clone(); + clone.AuthorName = "modified_author"; + clone.Role = ChatRole.Assistant; + clone.ResponseId = "modified_id"; + clone.ModelId = "modified_model"; + + // Assert - Original remains unchanged + Assert.Equal("original_author", original.AuthorName); + Assert.Equal(ChatRole.User, original.Role); + Assert.Equal("original_id", original.ResponseId); + Assert.Equal("original_model", original.ModelId); + + // Assert - Clone has modified values + Assert.Equal("modified_author", clone.AuthorName); + Assert.Equal(ChatRole.Assistant, clone.Role); + Assert.Equal("modified_id", clone.ResponseId); + Assert.Equal("modified_model", clone.ModelId); + } + + [Fact] + public void Clone_ModifyingSharedReferences_AffectsBothInstances() + { + // Arrange + var sharedAdditionalProperties = new AdditionalPropertiesDictionary { ["initial"] = "value" }; + var sharedContents = new List { new TextContent("initial") }; + + var original = new ChatResponseUpdate + { + AdditionalProperties = sharedAdditionalProperties, + Contents = sharedContents + }; + + // Act + var clone = original.Clone(); + + // Modify the shared reference objects + sharedAdditionalProperties["modified"] = "new_value"; + sharedContents.Add(new TextContent("added")); + + // Assert - Both original and clone are affected due to shallow copy + Assert.Same(original.AdditionalProperties, clone.AdditionalProperties); + Assert.Same(original.Contents, clone.Contents); + Assert.Equal(2, original.AdditionalProperties.Count); + Assert.Equal(2, clone.AdditionalProperties?.Count); + Assert.Equal(2, original.Contents.Count); + Assert.Equal(2, clone.Contents.Count); + Assert.True(original.AdditionalProperties.ContainsKey("modified")); + Assert.True(clone.AdditionalProperties?.ContainsKey("modified")); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..3a05f7fba9c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI; + +/// +/// Azure AI Inference-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with Azure AI Inference chat client implementation. +/// +public class AzureAIInferenceImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetChatCompletionsClient() + ?.AsIChatClient(TestRunnerConfiguration.Instance["AzureAIInference:ChatModel"] ?? "gpt-4o-mini"); +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..2cbdcd96abf --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,448 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.TestUtilities; +using Xunit; + +#pragma warning disable CA2000 // Dispose objects before losing scope +#pragma warning disable CA2214 // Do not call overridable methods in constructors + +namespace Microsoft.Extensions.AI; + +/// +/// Abstract base class for integration tests that verify ImageGeneratingChatClient with real IChatClient implementations. +/// Concrete test classes should inherit from this and provide a real IChatClient that supports function calling. +/// +public abstract class ImageGeneratingChatClientIntegrationTests : IDisposable +{ + private const string ImageKey = "meai_image"; + private readonly IChatClient? _baseChatClient; + + protected ImageGeneratingChatClientIntegrationTests() + { + _baseChatClient = CreateChatClient(); + ImageGenerator = new(); + + if (_baseChatClient != null) + { + ChatClient = _baseChatClient + .AsBuilder() + .UseImageGeneration(ImageGenerator) + .UseFunctionInvocation() + .Build(); + } + } + + /// Gets the ImageGeneratingChatClient configured with function invocation support. + protected IChatClient? ChatClient { get; } + + /// Gets the IImageGenerator used for testing. + protected CapturingImageGenerator ImageGenerator { get; } + + public void Dispose() + { + ChatClient?.Dispose(); + _baseChatClient?.Dispose(); + ImageGenerator.Dispose(); + GC.SuppressFinalize(this); + } + + /// + /// Creates the base IChatClient implementation to test with. + /// Should return a real chat client that supports function calling. + /// + /// An IChatClient instance, or null to skip tests. + protected abstract IChatClient? CreateChatClient(); + + /// + /// Helper method to get a chat response using either streaming or non-streaming based on the parameter. + /// + /// Whether to use streaming or non-streaming response. + /// The chat messages to send. + /// The chat options to use. + /// A ChatResponse from either streaming or non-streaming call. + protected async Task GetResponseAsync(bool useStreaming, IEnumerable messages, ChatOptions? options = null, IChatClient? chatClient = null) + { + chatClient ??= ChatClient ?? throw new InvalidOperationException("ChatClient is not initialized."); + + if (useStreaming) + { + return ValidateChatResponse(await chatClient.GetStreamingResponseAsync(messages, options).ToChatResponseAsync()); + } + else + { + return ValidateChatResponse(await chatClient.GetResponseAsync(messages, options)); + } + + static ChatResponse ValidateChatResponse(ChatResponse response) + { + var contents = response.Messages.SelectMany(m => m.Contents).ToArray(); + + List imageIds = []; + foreach (var toolResult in contents.OfType()) + { + Assert.NotNull(toolResult.Outputs); + + foreach (var dataContent in toolResult.Outputs.OfType()) + { + var imageId = dataContent.AdditionalProperties?[ImageKey] as string; + Assert.NotNull(imageId); + imageIds.Add(imageId); + } + } + + foreach (var textContent in contents.OfType()) + { + Assert.DoesNotContain(ImageKey, textContent.Text, StringComparison.OrdinalIgnoreCase); + foreach (var imageId in imageIds) + { + // Ensure no image IDs appear in text content + Assert.DoesNotContain(imageId, textContent.Text, StringComparison.OrdinalIgnoreCase); + } + } + + return response; + } + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task GenerateImage_CallsGenerateFunction_ReturnsDataContent(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await GetResponseAsync(useStreaming, + [new ChatMessage(ChatRole.User, "Please generate an image of a cat")], + chatOptions); + + // Assert + Assert.Single(imageGenerator.GenerateCalls); + var (request, _) = imageGenerator.GenerateCalls[0]; + Assert.Contains("cat", request.Prompt, StringComparison.OrdinalIgnoreCase); + Assert.Null(request.OriginalImages); // Generation, not editing + + // Verify that we get ImageGenerationToolResultContent back in the response + var imageResults = response.Messages + .SelectMany(m => m.Contents) + .OfType(); + + var imageResult = Assert.Single(imageResults); + Assert.NotNull(imageResult.Outputs); + var imageContent = Assert.Single(imageResult.Outputs.OfType()); + Assert.Equal("image/png", imageContent.MediaType); + Assert.False(imageContent.Data.IsEmpty); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task EditImage_WithImageInSameRequest_PassesExactDataContent(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var testImageData = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG header + var originalImageData = new DataContent(testImageData, "image/png") { Name = "original.png" }; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await GetResponseAsync(useStreaming, + [new ChatMessage(ChatRole.User, [new TextContent("Please edit this image to add a red border"), originalImageData])], + chatOptions); + + // Assert + var (request, _) = Assert.Single(imageGenerator.GenerateCalls); + Assert.NotNull(request.OriginalImages); + + var originalImage = Assert.Single(request.OriginalImages); + var originalImageContent = Assert.IsType(originalImage); + Assert.Equal(testImageData, originalImageContent.Data.ToArray()); + Assert.Equal("image/png", originalImageContent.MediaType); + Assert.Equal("original.png", originalImageContent.Name); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task GenerateThenEdit_FromChatHistory_EditsGeneratedImage(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a dog") + }; + + // First request: Generate image + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + + // Second request: Edit the generated image + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to make it more colorful")); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + + // Assert + Assert.Equal(2, imageGenerator.GenerateCalls.Count); + + // First call should be generation (no original images) + var (firstRequest, _) = imageGenerator.GenerateCalls[0]; + Assert.Null(firstRequest.OriginalImages); + + // Extract the DataContent from the ImageGenerationToolResultContent + var firstToolResultContent = Assert.Single(firstResponse.Messages.SelectMany(m => m.Contents).OfType()); + Assert.NotNull(firstToolResultContent.Outputs); + var firstContent = Assert.Single(firstToolResultContent.Outputs.OfType()); + + // Second call should be editing (with original images) + var (secondRequest, _) = imageGenerator.GenerateCalls[1]; + Assert.Single(secondResponse.Messages.SelectMany(m => m.Contents).OfType().SelectMany(t => t.Outputs!.OfType())); + Assert.NotNull(secondRequest.OriginalImages); + var editContent = Assert.Single(secondRequest.OriginalImages); + Assert.Equal(firstContent, editContent); // Should be the same image as generated in first call + + var editedImage = Assert.IsType(secondRequest.OriginalImages.First()); + Assert.Equal("image/png", editedImage.MediaType); + Assert.Contains("generated_image_1", editedImage.Name); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task MultipleEdits_EditsLatestImage(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a tree") + }; + + // First: Generate image + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + + // Second: First edit + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add flowers")); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + chatHistory.AddRange(secondResponse.Messages); + + // Third: Second edit (should edit the latest version by default) + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit that last image to add birds")); + var thirdResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + + // Assert + Assert.Equal(3, imageGenerator.GenerateCalls.Count); + + // Third call should edit the second generated image (from first edit), not the original + var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; + Assert.NotNull(thirdRequest.OriginalImages); + + // Extract the DataContent from the second response's ImageGenerationToolResultContent + var secondToolResultContent = Assert.Single(secondResponse.Messages.SelectMany(m => m.Contents).OfType()); + var secondImage = Assert.Single(secondToolResultContent.Outputs!.OfType()); + var lastImageToEdit = Assert.Single(thirdRequest.OriginalImages.OfType()); + Assert.Equal(secondImage, lastImageToEdit); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task MultipleEdits_EditsFirstImage(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + var chatHistory = new List + { + new(ChatRole.User, "Please generate an image of a tree") + }; + + // First: Generate image + var firstResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + chatHistory.AddRange(firstResponse.Messages); + + // Second: First edit + chatHistory.Add(new ChatMessage(ChatRole.User, "Please edit the image to add fruit")); + var secondResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + chatHistory.AddRange(secondResponse.Messages); + + // Third: Second edit (should edit the latest version by default) + chatHistory.Add(new ChatMessage(ChatRole.User, "That didn't work out. Please edit the original image to add birds")); + var thirdResponse = await GetResponseAsync(useStreaming, chatHistory, chatOptions); + + // Assert + Assert.Equal(3, imageGenerator.GenerateCalls.Count); + + // Third call should edit the original generated image (not from edit) + var (thirdRequest, _) = imageGenerator.GenerateCalls[2]; + Assert.NotNull(thirdRequest.OriginalImages); + + // Extract the DataContent from the first response's ImageGenerationToolResultContent + var firstToolResultContent = Assert.Single(firstResponse.Messages.SelectMany(m => m.Contents).OfType()); + var firstGeneratedImage = Assert.Single(firstToolResultContent.Outputs!.OfType()); + var lastImageToEdit = Assert.IsType(thirdRequest.OriginalImages.First()); + Assert.Equal(firstGeneratedImage, lastImageToEdit); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task ImageGeneration_WithOptions_PassesOptionsToGenerator(bool useStreaming) + { + SkipIfNotEnabled(); + + var imageGenerator = ImageGenerator; + var imageGenerationOptions = new ImageGenerationOptions + { + Count = 2, + ImageSize = new System.Drawing.Size(512, 512) + }; + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool { Options = imageGenerationOptions }] + }; + + // Act + var response = await GetResponseAsync(useStreaming, + [new ChatMessage(ChatRole.User, "Generate an image of a castle")], + chatOptions); + + // Assert + Assert.Single(imageGenerator.GenerateCalls); + var (_, options) = imageGenerator.GenerateCalls[0]; + Assert.NotNull(options); + Assert.Equal(2, options.Count); + Assert.Equal(new System.Drawing.Size(512, 512), options.ImageSize); + } + + [ConditionalTheory] + [InlineData(false)] // Non-streaming + [InlineData(true)] // Streaming + public virtual async Task ImageContentHandling_AllImages_ReplacesImagesWithPlaceholders(bool useStreaming) + { + SkipIfNotEnabled(); + + var testImageData = new byte[] { 0x89, 0x50, 0x4E, 0x47 }; // PNG header + var capturedMessages = new List>(); + + // Create a new ImageGeneratingChatClient with AllImages data content handling + using var imageGeneratingClient = _baseChatClient! + .AsBuilder() + .UseImageGeneration(ImageGenerator) + .Use((messages, options, next, cancellationToken) => + { + capturedMessages.Add(messages); + return next(messages, options, cancellationToken); + }) + .UseFunctionInvocation() + .Build(); + + var originalImage = new DataContent(testImageData, "image/png") { Name = "test.png" }; + + // Act + await GetResponseAsync(useStreaming, + [ + new ChatMessage(ChatRole.User, + [ + new TextContent("Here's an image to process"), + originalImage + ]) + ], + new ChatOptions { Tools = [new HostedImageGenerationTool()] }, + imageGeneratingClient); + + // Assert + Assert.NotEmpty(capturedMessages); + var processedMessages = capturedMessages.First().ToList(); + var userMessage = processedMessages.First(m => m.Role == ChatRole.User); + + // Should have text content with placeholder instead of original image + var textContents = userMessage.Contents.OfType().ToList(); + Assert.Contains(textContents, tc => tc.Text.Contains(ImageKey) && tc.Text.Contains("] available for edit")); + + // Should not contain the original DataContent + Assert.DoesNotContain(userMessage.Contents, c => c == originalImage); + } + + /// + /// Test image generator that captures calls and returns fake image data. + /// + protected sealed class CapturingImageGenerator : IImageGenerator + { + private const string TestImageMediaType = "image/png"; + private static readonly byte[] _testImageData = [0x89, 0x50, 0x4E, 0x47]; // PNG header + + public List<(ImageGenerationRequest request, ImageGenerationOptions? options)> GenerateCalls { get; } = []; + public int ImageCounter { get; private set; } + + public Task GenerateAsync(ImageGenerationRequest request, ImageGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + GenerateCalls.Add((request, options)); + + // Create fake image data with unique content + var imageData = new byte[_testImageData.Length + 4]; + _testImageData.CopyTo(imageData, 0); + BitConverter.GetBytes(++ImageCounter).CopyTo(imageData, _testImageData.Length); + + var imageContent = new DataContent(imageData, TestImageMediaType) + { + Name = $"generated_image_{ImageCounter}.png" + }; + + return Task.FromResult(new ImageGenerationResponse([imageContent])); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + // No resources to dispose + } + } + + [MemberNotNull(nameof(ChatClient))] + protected void SkipIfNotEnabled() + { + string? skipIntegration = TestRunnerConfiguration.Instance["SkipIntegrationTests"]; + + if (skipIntegration is not null || ChatClient is null) + { + throw new SkipTestException("Client is not enabled."); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..7a6f60af778 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OllamaSharp.Integration.Tests/OllamaSharpImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using OllamaSharp; + +namespace Microsoft.Extensions.AI; + +/// +/// OllamaSharp-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with OllamaSharp chat client implementation. +/// +public class OllamaSharpImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetOllamaUri() is Uri endpoint ? + new OllamaApiClient(endpoint, "llama3.2") : + null; +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs new file mode 100644 index 00000000000..7f9e6195aa3 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIImageGeneratingChatClientIntegrationTests.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI; + +/// +/// OpenAI-specific integration tests for ImageGeneratingChatClient. +/// Tests the ImageGeneratingChatClient with OpenAI's chat client implementation. +/// +public class OpenAIImageGeneratingChatClientIntegrationTests : ImageGeneratingChatClientIntegrationTests +{ + protected override IChatClient? CreateChatClient() => + IntegrationTestHelpers.GetOpenAIClient() + ?.GetChatClient(TestRunnerConfiguration.Instance["OpenAI:ChatModel"] ?? "gpt-4o-mini").AsIChatClient(); +} diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs index 53997c790e0..1ee738bd6a0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientTests.cs @@ -4640,6 +4640,356 @@ public async Task ResponseWithRefusalContent_ParsesCorrectly() Assert.Equal("Refusal", errorContent.ErrorCode); } + [Fact] + public async Task HostedImageGenerationTool_NonStreaming() + { + const string Input = """ + { + "model": "gpt-4o", + "tools": [ + { + "type": "image_generation", + "model": "gpt-image-1", + "size": "1024x1024", + "output_format": "png" + } + ], + "tool_choice": "auto", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Generate an image of a cat" + } + ] + } + ] + } + """; + + const string Output = """ + { + "id": "resp_abc123", + "object": "response", + "created_at": 1741891428, + "status": "completed", + "model": "gpt-4o-2024-11-20", + "output": [ + { + "type": "image_generation_call", + "id": "img_call_abc123", + "result": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + } + ], + "usage": { + "input_tokens": 15, + "output_tokens": 0, + "total_tokens": 15 + } + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o"); + + var imageTool = new HostedImageGenerationTool + { + Options = new ImageGenerationOptions + { + ModelId = "gpt-image-1", + ImageSize = new(1024, 1024), + MediaType = "image/png" + } + }; + var response = await client.GetResponseAsync("Generate an image of a cat", new ChatOptions + { + Tools = [imageTool] + }); + + Assert.NotNull(response); + Assert.Single(response.Messages); + Assert.Equal(ChatRole.Assistant, response.Messages[0].Role); + + var contents = response.Messages[0].Contents; + Assert.Equal(2, contents.Count); + + // First content should be the tool call + var toolCall = contents[0] as ImageGenerationToolCallContent; + Assert.NotNull(toolCall); + Assert.Equal("img_call_abc123", toolCall.ImageId); + + // Second content should be the result with image data + var toolResult = contents[1] as ImageGenerationToolResultContent; + Assert.NotNull(toolResult); + Assert.Equal("img_call_abc123", toolResult.ImageId); + Assert.Single(toolResult.Outputs!); + + var imageData = toolResult.Outputs![0] as DataContent; + Assert.NotNull(imageData); + Assert.Equal("image/png", imageData.MediaType); + Assert.True(imageData.Data.Length > 0); + } + + [Fact] + public async Task HostedImageGenerationTool_Streaming() + { + const string Input = """ + { + "model": "gpt-4o", + "tools": [ + { + "type": "image_generation", + "model": "gpt-image-1", + "size": "1024x1024", + "output_format": "png" + } + ], + "tool_choice": "auto", + "stream": true, + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Generate an image of a dog" + } + ] + } + ] + } + """; + + const string Output = """ + event: response.created + data: {"type":"response.created","response":{"id":"resp_def456","object":"response","created_at":1741892091,"status":"in_progress","model":"gpt-4o-2024-11-20","output":[],"tools":[{"type":"image_generation","image_generation":{"model":"gpt-image-1","size":{"width":1024,"height":1024},"output_format":"png"}}]}} + + event: response.in_progress + data: {"type":"response.in_progress","response":{"id":"resp_def456","object":"response","created_at":1741892091,"status":"in_progress","model":"gpt-4o-2024-11-20","output":[]}} + + event: response.output_item.added + data: {"type":"response.output_item.added","output_index":0,"item":{"type":"image_generation_call","id":"img_call_def456","status":"in_progress"}} + + event: response.image_generation_call.in_progress + data: {"type":"response.image_generation_call.in_progress","item_id":"img_call_def456","output_index":0} + + event: response.image_generation_call.generating + data: {"type":"response.image_generation_call.generating","item_id":"img_call_def456","output_index":0} + + event: response.image_generation_call.partial_image + data: {"type":"response.image_generation_call.partial_image","item_id":"img_call_def456","output_index":0,"partial_image_index":0,"partial_image_b64":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="} + + event: response.output_item.done + data: {"type":"response.output_item.done","output_index":0,"item":{"type":"image_generation_call","id":"img_call_def456","image_result_b64":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="}} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_def456","object":"response","created_at":1741892091,"status":"completed","model":"gpt-4o-2024-11-20","output":[{"type":"image_generation_call","id":"img_call_def456","image_result_b64":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="}],"usage":{"input_tokens":15,"output_tokens":0,"total_tokens":15}}} + + + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o"); + + List updates = []; + var imageTool = new HostedImageGenerationTool + { + Options = new ImageGenerationOptions + { + ModelId = "gpt-image-1", + ImageSize = new(1024, 1024), + MediaType = "image/png" + } + }; + await foreach (var update in client.GetStreamingResponseAsync("Generate an image of a dog", new ChatOptions + { + Tools = [imageTool] + })) + { + updates.Add(update); + } + + Assert.True(updates.Count >= 6); + + // Should have updates for: created, in_progress, tool call start, generating, partial image, completion + var createdUpdate = updates.First(u => u.CreatedAt.HasValue); + Assert.Equal("resp_def456", createdUpdate.ResponseId); + Assert.Equal("gpt-4o-2024-11-20", createdUpdate.ModelId); + + // Should have tool call content + var toolCallUpdate = updates.FirstOrDefault(u => + u.Contents != null && u.Contents.Any(c => c is ImageGenerationToolCallContent)); + Assert.NotNull(toolCallUpdate); + var toolCall = toolCallUpdate.Contents.OfType().First(); + Assert.Equal("img_call_def456", toolCall.ImageId); + + // Should have partial image content + var partialImageUpdate = updates.FirstOrDefault(u => + u.Contents != null && u.Contents.Any(c => c is ImageGenerationToolResultContent result && + result.Outputs != null && result.Outputs.Any(o => o.AdditionalProperties != null && o.AdditionalProperties.ContainsKey("PartialImageIndex")))); + Assert.NotNull(partialImageUpdate); + + // Should have final completion with usage + var completionUpdate = updates.FirstOrDefault(u => + u.Contents != null && u.Contents.Any(c => c is UsageContent)); + Assert.NotNull(completionUpdate); + var usage = completionUpdate.Contents.OfType().First(); + Assert.Equal(15, usage.Details.InputTokenCount); + Assert.Equal(0, usage.Details.OutputTokenCount); + Assert.Equal(15, usage.Details.TotalTokenCount); + } + + [Fact] + public async Task HostedImageGenerationTool_StreamingMultipleImages() + { + const string Input = """ + { + "model": "gpt-4o", + "tools": [ + { + "type": "image_generation", + "model": "gpt-image-1", + "size": "512x512", + "output_format": "webp", + "partial_images": 3 + } + ], + "tool_choice": "auto", + "stream": true, + "input": [ + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": "Generate an image of a sunset" + } + ] + } + ] + } + """; + + const string Output = """ + event: response.created + data: {"type":"response.created","response":{"id":"resp_ghi789","object":"response","created_at":1741892091,"status":"in_progress","model":"gpt-4o-2024-11-20","output":[],"tools":[{"type":"image_generation","image_generation":{"model":"gpt-image-1","size":{"width":512,"height":512},"output_format":"webp","partial_images":3}}]}} + + event: response.in_progress + data: {"type":"response.in_progress","response":{"id":"resp_ghi789","object":"response","created_at":1741892091,"status":"in_progress","model":"gpt-4o-2024-11-20","output":[]}} + + event: response.output_item.added + data: {"type":"response.output_item.added","output_index":0,"item":{"type":"image_generation_call","id":"img_call_ghi789","status":"in_progress"}} + + event: response.image_generation_call.in_progress + data: {"type":"response.image_generation_call.in_progress","item_id":"img_call_ghi789","output_index":0} + + event: response.image_generation_call.generating + data: {"type":"response.image_generation_call.generating","item_id":"img_call_ghi789","output_index":0} + + event: response.image_generation_call.partial_image + data: {"type":"response.image_generation_call.partial_image","item_id":"img_call_ghi789","output_index":0,"partial_image_index":0,"partial_image_b64":"SGVsbG8x"} + + event: response.image_generation_call.partial_image + data: {"type":"response.image_generation_call.partial_image","item_id":"img_call_ghi789","output_index":0,"partial_image_index":1,"partial_image_b64":"SGVsbG8y"} + + event: response.image_generation_call.partial_image + data: {"type":"response.image_generation_call.partial_image","item_id":"img_call_ghi789","output_index":0,"partial_image_index":2,"partial_image_b64":"SGVsbG8z"} + + event: response.output_item.done + data: {"type":"response.output_item.done","output_index":0,"item":{"type":"image_generation_call","id":"img_call_ghi789","image_result_b64":"SGVsbG8z"}} + + event: response.completed + data: {"type":"response.completed","response":{"id":"resp_ghi789","object":"response","created_at":1741892091,"status":"completed","model":"gpt-4o-2024-11-20","output":[{"type":"image_generation_call","id":"img_call_ghi789","image_result_b64":"SGVsbG8z"}],"usage":{"input_tokens":18,"output_tokens":0,"total_tokens":18}}} + + + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateResponseClient(httpClient, "gpt-4o"); + + List updates = []; + var imageTool = new HostedImageGenerationTool + { + Options = new ImageGenerationOptions + { + ModelId = "gpt-image-1", + ImageSize = new(512, 512), + MediaType = "image/webp", + StreamingCount = 3 + } + }; + await foreach (var update in client.GetStreamingResponseAsync("Generate an image of a sunset", new ChatOptions + { + Tools = [imageTool] + })) + { + updates.Add(update); + } + + Assert.True(updates.Count >= 8); // Should have multiple partial image updates plus generating event + + // Should have multiple partial images with different indices + var partialImageUpdates = updates.Where(u => + u.Contents != null && u.Contents.Any(c => c is ImageGenerationToolResultContent result && + result.Outputs != null && result.Outputs.Any(o => o.AdditionalProperties != null && o.AdditionalProperties.ContainsKey("PartialImageIndex")))).ToList(); + + Assert.True(partialImageUpdates.Count >= 3); + + // Verify partial images have correct indices and WebP format + for (int i = 0; i < 3; i++) + { + var partialUpdate = partialImageUpdates.FirstOrDefault(u => + u.Contents.OfType().Any(result => + HasPartialImageWithIndex(result, i))); + Assert.NotNull(partialUpdate); + } + + static bool HasPartialImageWithIndex(ImageGenerationToolResultContent result, int index) + { + if (result.Outputs == null) + { + return false; + } + + return result.Outputs.Any(o => HasCorrectImageData(o, index)); + } + + static bool HasCorrectImageData(AIContent o, int index) + { + if (o.AdditionalProperties == null) + { + return false; + } + + if (!o.AdditionalProperties.TryGetValue("PartialImageIndex", out var imageIndex)) + { + return false; + } + + if (imageIndex == null || !imageIndex.Equals(index)) + { + return false; + } + + return o is DataContent dataContent && dataContent.MediaType == "image/webp"; + } + + // Verify tool call uses correct settings + var toolCallUpdate = updates.FirstOrDefault(u => + u.Contents != null && u.Contents.Any(c => c is ImageGenerationToolCallContent)); + Assert.NotNull(toolCallUpdate); + var toolCall = toolCallUpdate.Contents.OfType().First(); + Assert.Equal("img_call_ghi789", toolCall.ImageId); + } + private static IChatClient CreateResponseClient(HttpClient httpClient, string modelId) => new OpenAIClient( new ApiKeyCredential("apikey"), diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs new file mode 100644 index 00000000000..a70d19abffc --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ImageGeneratingChatClientTests.cs @@ -0,0 +1,386 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class ImageGeneratingChatClientTests +{ + [Fact] + public void ImageGeneratingChatClient_InvalidArgs_Throws() + { + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + + Assert.Throws("innerClient", () => new ImageGeneratingChatClient(null!, imageGenerator)); + Assert.Throws("imageGenerator", () => new ImageGeneratingChatClient(innerClient, null!)); + } + + [Fact] + public void UseImageGeneration_WithNullBuilder_Throws() + { + Assert.Throws("builder", () => ((ChatClientBuilder)null!).UseImageGeneration()); + } + + [Fact] + public async Task GetResponseAsync_WithoutImageGenerationTool_PassesThrough() + { + // Arrange + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [AIFunctionFactory.Create(() => "dummy function", name: "DummyFunction")] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.NotNull(response); + Assert.Equal("test response", response.Messages[0].Text); + + // Verify that tools collection still has the original function, not replaced + Assert.Single(chatOptions.Tools); + Assert.IsAssignableFrom(chatOptions.Tools[0]); + } + + [Fact] + public async Task GetResponseAsync_WithImageGenerationTool_ReplacesTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(3, capturedOptions.Tools.Count); + + // Verify the functions are properly created + var generateImageFunction = capturedOptions.Tools[0] as AIFunction; + var editImageFunction = capturedOptions.Tools[1] as AIFunction; + var getImagesForEditImageFunction = capturedOptions.Tools[2] as AIFunction; + + Assert.NotNull(generateImageFunction); + Assert.NotNull(editImageFunction); + Assert.NotNull(getImagesForEditImageFunction); + Assert.Equal("GenerateImage", generateImageFunction.Name); + Assert.Equal("EditImage", editImageFunction.Name); + Assert.Equal("GetImagesForEdit", getImagesForEditImageFunction.Name); + } + + [Fact] + public async Task GetResponseAsync_WithMixedTools_ReplacesOnlyImageGenerationTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var dummyFunction = AIFunctionFactory.Create(() => "dummy", name: "DummyFunction"); + var chatOptions = new ChatOptions + { + Tools = [dummyFunction, new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(4, capturedOptions.Tools.Count); // DummyFunction + GenerateImage + EditImage + GetImagesForEdit + + Assert.Same(dummyFunction, capturedOptions.Tools[0]); // Original function preserved + Assert.IsAssignableFrom(capturedOptions.Tools[1]); // GenerateImage function + Assert.IsAssignableFrom(capturedOptions.Tools[2]); // EditImage function + } + + [Fact] + public void UseImageGeneration_ServiceProviderIntegration_Works() + { + // Arrange + var services = new ServiceCollection(); + services.AddSingleton(); + + using var serviceProvider = services.BuildServiceProvider(); + using var innerClient = new TestChatClient(); + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration() + .Build(serviceProvider); + + // Assert + Assert.IsType(client); + } + + [Fact] + public void UseImageGeneration_WithProvidedImageGenerator_Works() + { + // Arrange + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration(imageGenerator) + .Build(); + + // Assert + Assert.IsType(client); + } + + [Fact] + public void UseImageGeneration_WithConfigureCallback_CallsCallback() + { + // Arrange + using var innerClient = new TestChatClient(); + using var imageGenerator = new TestImageGenerator(); + bool configureCallbackInvoked = false; + + // Act + using var client = innerClient + .AsBuilder() + .UseImageGeneration(imageGenerator, configure: c => + { + Assert.NotNull(c); + configureCallbackInvoked = true; + }) + .Build(); + + // Assert + Assert.True(configureCallbackInvoked); + } + + [Fact] + public async Task GetStreamingResponseAsync_WithImageGenerationTool_ReplacesTool() + { + // Arrange + bool innerClientCalled = false; + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => + { + innerClientCalled = true; + capturedOptions = options; + return GetUpdatesAsync(); + } + }; + + static async IAsyncEnumerable GetUpdatesAsync() + { + await Task.Yield(); + yield return new(ChatRole.Assistant, "test"); + } + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + await foreach (var update in client.GetStreamingResponseAsync([new(ChatRole.User, "test")], chatOptions)) + { + // Process updates + } + + // Assert + Assert.True(innerClientCalled); + Assert.NotNull(capturedOptions); + Assert.NotNull(capturedOptions.Tools); + Assert.Equal(3, capturedOptions.Tools.Count); + } + + [Fact] + public async Task GetResponseAsync_WithNullOptions_DoesNotThrow() + { + // Arrange + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + // Act & Assert + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], null); + Assert.NotNull(response); + } + + [Fact] + public async Task GetResponseAsync_WithEmptyTools_DoesNotModify() + { + // Arrange + ChatOptions? capturedOptions = null; + + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + capturedOptions = options; + return Task.FromResult(new ChatResponse(new ChatMessage(ChatRole.Assistant, "test response"))); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [] + }; + + // Act + await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.Same(chatOptions, capturedOptions); +#pragma warning disable CA1508 + Assert.NotNull(capturedOptions?.Tools); +#pragma warning restore CA1508 + Assert.Empty(capturedOptions.Tools); + } + + [Fact] + public async Task GetResponseAsync_WithFunctionCallContent_ReplacesWithImageGenerationToolCallContent() + { + // Arrange + var callId = "test-call-id"; + using var innerClient = new TestChatClient + { + GetResponseAsyncCallback = (messages, options, cancellationToken) => + { + var responseMessage = new ChatMessage(ChatRole.Assistant, + [new FunctionCallContent(callId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" })]); + return Task.FromResult(new ChatResponse(responseMessage)); + }, + }; + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var response = await client.GetResponseAsync([new(ChatRole.User, "test")], chatOptions); + + // Assert + Assert.NotNull(response); + Assert.Single(response.Messages); + var message = response.Messages[0]; + Assert.Single(message.Contents); + + var imageToolCallContent = Assert.IsType(message.Contents[0]); + Assert.Equal(callId, imageToolCallContent.ImageId); + } + + [Fact] + public async Task GetStreamingResponseAsync_WithFunctionCallContent_ReplacesWithImageGenerationToolCallContent() + { + // Arrange + var callId = "test-call-id"; + using var innerClient = new TestChatClient + { + GetStreamingResponseAsyncCallback = (messages, options, cancellationToken) => + { + return GetUpdatesAsync(); + } + }; + + async IAsyncEnumerable GetUpdatesAsync() + { + await Task.Yield(); + yield return new ChatResponseUpdate(ChatRole.Assistant, + [new FunctionCallContent(callId, "GenerateImage", new Dictionary { ["prompt"] = "a cat" })]); + } + + using var imageGenerator = new TestImageGenerator(); + using var client = new ImageGeneratingChatClient(innerClient, imageGenerator); + + var chatOptions = new ChatOptions + { + Tools = [new HostedImageGenerationTool()] + }; + + // Act + var updates = new List(); + await foreach (var responseUpdate in client.GetStreamingResponseAsync([new(ChatRole.User, "test")], chatOptions)) + { + updates.Add(responseUpdate); + } + + // Assert + Assert.Single(updates); + var update = updates[0]; + Assert.Single(update.Contents); + + var imageToolCallContent = Assert.IsType(update.Contents[0]); + Assert.Equal(callId, imageToolCallContent.ImageId); + } +} diff --git a/test/TestUtilities/XUnit/ConditionalTheoryDiscoverer.cs b/test/TestUtilities/XUnit/ConditionalTheoryDiscoverer.cs index b1e53b8ed77..e30b5206c8c 100644 --- a/test/TestUtilities/XUnit/ConditionalTheoryDiscoverer.cs +++ b/test/TestUtilities/XUnit/ConditionalTheoryDiscoverer.cs @@ -63,9 +63,21 @@ protected override IEnumerable CreateTestCasesForDataRow(ITestFr } } - return skipReason != null ? - base.CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason) - : base.CreateTestCasesForDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow); + if (skipReason != null) + { + return base.CreateTestCasesForSkippedDataRow(discoveryOptions, testMethod, theoryAttribute, dataRow, skipReason); + } + + // Create test cases that can handle runtime SkipTestException + return new[] + { + new SkippedTheoryTestCase( + DiagnosticMessageSink, + discoveryOptions.MethodDisplayOrDefault(), + discoveryOptions.MethodDisplayOptionsOrDefault(), + testMethod, + dataRow) + }; } protected override IEnumerable CreateTestCasesForSkippedDataRow( diff --git a/test/TestUtilities/XUnit/SkippedTheoryTestCase.cs b/test/TestUtilities/XUnit/SkippedTheoryTestCase.cs new file mode 100644 index 00000000000..e91a8f762d5 --- /dev/null +++ b/test/TestUtilities/XUnit/SkippedTheoryTestCase.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.TestUtilities; + +/// +/// A test case for ConditionalTheory that can handle runtime SkipTestException +/// by wrapping the message bus with SkippedTestMessageBus. +/// +public class SkippedTheoryTestCase : XunitTestCase +{ + [Obsolete("Called by the de-serializer; should only be called by deriving classes for de-serialization purposes", error: true)] + public SkippedTheoryTestCase() + { + } + + public SkippedTheoryTestCase( + IMessageSink diagnosticMessageSink, + TestMethodDisplay defaultMethodDisplay, + TestMethodDisplayOptions defaultMethodDisplayOptions, + ITestMethod testMethod, + object[]? testMethodArguments = null) + : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod, testMethodArguments) + { + } + + public override async Task RunAsync(IMessageSink diagnosticMessageSink, + IMessageBus messageBus, + object[] constructorArguments, + ExceptionAggregator aggregator, + CancellationTokenSource cancellationTokenSource) + { + using SkippedTestMessageBus skipMessageBus = new(messageBus); + var result = await base.RunAsync(diagnosticMessageSink, skipMessageBus, constructorArguments, aggregator, cancellationTokenSource); + if (skipMessageBus.SkippedTestCount > 0) + { + result.Failed -= skipMessageBus.SkippedTestCount; + result.Skipped += skipMessageBus.SkippedTestCount; + } + + return result; + } +} \ No newline at end of file