diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputRequest.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputRequest.cs index c9fbe20b8f7..fff360570f3 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputRequest.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputRequest.cs @@ -38,7 +38,9 @@ internal ExternalInputRequest(string text) /// requireApproval: true) over so that /// hosts which speak the approval protocol see the approval-bearing content. /// - AIContent? IExternalRequestEnvelope.GetInnerRequestContent() + AIContent? IExternalRequestEnvelope.GetInnerRequestContent() => this.GetInnerRequestContent(); + + private AIContent? GetInnerRequestContent() { IList? messages = this.AgentResponse?.Messages; if (messages is null) @@ -73,5 +75,16 @@ internal ExternalInputRequest(string text) /// object IExternalRequestEnvelope.CreateResponse(IList messages) - => new ExternalInputResponse(messages); + => new ExternalInputResponse(messages) + { + RequestId = GetContentId(this.GetInnerRequestContent()), + }; + + private static string? GetContentId(AIContent? content) + => content switch + { + ToolCallContent functionCall => functionCall.CallId, + InputRequestContent toolApprovalRequest => toolApprovalRequest.RequestId, + _ => null, + }; } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputResponse.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputResponse.cs index 0653a12ce5b..fe13433b3ae 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputResponse.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/Events/ExternalInputResponse.cs @@ -16,6 +16,12 @@ public sealed class ExternalInputResponse /// public IList Messages { get; } + /// + /// Gets the request id that produced this response, if available. + /// + [JsonInclude] + public string? RequestId { get; internal init; } + internal bool HasMessages => this.Messages?.Count > 0; /// @@ -23,8 +29,8 @@ public sealed class ExternalInputResponse /// /// The external input message being provided to the workflow. public ExternalInputResponse(ChatMessage message) + : this([message], requestId: null) { - this.Messages = [message]; } /// @@ -33,7 +39,14 @@ public ExternalInputResponse(ChatMessage message) /// The external input messages being provided to the workflow. [JsonConstructor] public ExternalInputResponse(IList messages) + : this(messages, requestId: null) + { + } + + internal ExternalInputResponse(IList messages, string? requestId) { this.Messages = messages; + this.RequestId = requestId; } + } diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/InvokeFunctionToolExecutor.cs b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/InvokeFunctionToolExecutor.cs index 386c3712b95..20699ee4aa3 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/InvokeFunctionToolExecutor.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/ObjectModel/InvokeFunctionToolExecutor.cs @@ -29,6 +29,7 @@ internal sealed class InvokeFunctionToolExecutor( DeclarativeActionExecutor(model, state) { private const string ApprovalSnapshotStateKey = nameof(_approvalSnapshots); + private const string HasApprovalRequiredInvocationStateKey = nameof(_hasApprovalRequiredInvocation); private const string PendingCallIdsStateKey = nameof(_pendingNonApprovalCallIds); private const string LegacyApprovalSnapshotStateKey = "_approvalSnapshot"; @@ -39,6 +40,8 @@ internal sealed class InvokeFunctionToolExecutor( /// private readonly ConcurrentDictionary _approvalSnapshots = new(StringComparer.Ordinal); + private bool _hasApprovalRequiredInvocation; + /// /// Per-invocation call ids for in-flight non-approval requests; used to match the /// returning on the response path. Used as a set; @@ -94,6 +97,7 @@ public static class Steps // Capture the evaluated parameters keyed by request id; the matching response // resumes from this snapshot. this._approvalSnapshots[requestId] = new ApprovalSnapshot(functionName, arguments); + this._hasApprovalRequiredInvocation = true; requestMessage.Contents.Add(new ToolApprovalRequestContent(requestId, functionCall)); } @@ -125,24 +129,29 @@ public async ValueTask CaptureResponseAsync( { bool autoSend = this.GetAutoSendValue(); string? conversationId = this.GetConversationId(); + string? responseRequestId = response.RequestId; // Match the inbound result by its per-invocation call id. FunctionResultContent? matchingResult = response.Messages .SelectMany(m => m.Contents) .OfType() - .FirstOrDefault(r => this.IsKnownPendingId(r.CallId)); + .FirstOrDefault(r => this.IsCorrelatedPendingNonApprovalResult(r, responseRequestId)); + FunctionResultContent? approvedFunctionResult = null; - // Legacy non-approval backstop: when no pendings are tracked, accept a result - // whose CallId equals this.Id. The runtime has already routed the response to - // this executor's port and the framework does not invoke a function here. + // Legacy non-approval backstop: when no pendings are tracked and approval is + // not required, accept a result whose CallId equals this.Id. The runtime has + // already routed the response to this executor's port and the framework does + // not invoke a function here. if (matchingResult is null + && !this._hasApprovalRequiredInvocation && this._pendingNonApprovalCallIds.IsEmpty && this._approvalSnapshots.IsEmpty) { matchingResult = response.Messages .SelectMany(m => m.Contents) .OfType() - .FirstOrDefault(r => string.Equals(r.CallId, this.Id, StringComparison.Ordinal)); + .FirstOrDefault(r => string.Equals(r.CallId, this.Id, StringComparison.Ordinal) + && IsCorrelatedResponse(r.CallId, responseRequestId)); } // When the caller approved an approval-required function call but didn't execute it @@ -160,8 +169,9 @@ public async ValueTask CaptureResponseAsync( // Prefer an approval matching a pending snapshot; otherwise take the first // present approval. ToolApprovalResponseContent? approval = - approvals.FirstOrDefault(r => this._approvalSnapshots.ContainsKey(r.RequestId)) - ?? approvals.FirstOrDefault(); + approvals.FirstOrDefault(r => this._approvalSnapshots.ContainsKey(r.RequestId) + && IsCorrelatedResponse(r.RequestId, responseRequestId)) + ?? approvals.FirstOrDefault(r => IsCorrelatedResponse(r.RequestId, responseRequestId)); if (approval is not null) { @@ -177,12 +187,17 @@ public async ValueTask CaptureResponseAsync( else if (this._approvalSnapshots.TryRemove(approval.RequestId, out ApprovalSnapshot? snapshot)) { matchingResult = await this.InvokeRegisteredFunctionAsync(approval.RequestId, snapshot, cancellationToken).ConfigureAwait(false); + approvedFunctionResult = matchingResult; } else { await this.AssignErrorAsync(context, "No pending approval matched the response.").ConfigureAwait(false); } } + else if (!this._approvalSnapshots.IsEmpty) + { + await this.AssignErrorAsync(context, "No pending approval matched the response.").ConfigureAwait(false); + } } if (matchingResult is not null) @@ -196,16 +211,15 @@ public async ValueTask CaptureResponseAsync( AgentResponse resultResponse = new([new ChatMessage(ChatRole.Tool, [matchingResult])]); await context.AddEventAsync(new AgentResponseEvent(this.Id, resultResponse), cancellationToken).ConfigureAwait(false); } - - // Drop the per-invocation entry now that the response has been processed. - this._pendingNonApprovalCallIds.TryRemove(matchingResult.CallId, out _); - this._approvalSnapshots.TryRemove(matchingResult.CallId, out _); } // Store messages if output path is configured if (this.Model.Output?.Messages is not null) { - await this.AssignAsync(this.Model.Output.Messages?.Path, response.Messages.ToFormula(), context).ConfigureAwait(false); + await this.AssignAsync( + this.Model.Output.Messages?.Path, + this.GetSideEffectMessages(response).ToFormula(), + context).ConfigureAwait(false); } // Add messages to conversation if conversationId is provided @@ -214,37 +228,60 @@ public async ValueTask CaptureResponseAsync( // actual AI-generated tool calls and would be rejected by the API. if (conversationId is not null) { - foreach (ChatMessage message in TransformConversationMessages(response.Messages)) + IEnumerable sideEffectMessages = this.GetSideEffectMessages(response); + if (approvedFunctionResult is not null) + { + sideEffectMessages = sideEffectMessages.Append(new ChatMessage(ChatRole.Tool, [approvedFunctionResult])); + } + + foreach (ChatMessage message in TransformConversationMessages(sideEffectMessages)) { await agentProvider.CreateMessageAsync(conversationId, message, cancellationToken).ConfigureAwait(false); } } + if (matchingResult is not null) + { + // Drop the per-invocation entry now that the response side effects have been processed. + this._pendingNonApprovalCallIds.TryRemove(matchingResult.CallId, out _); + this._approvalSnapshots.TryRemove(matchingResult.CallId, out _); + } + // Completes the action after processing the function result. await context.RaiseCompletionEventAsync(this.Model, cancellationToken).ConfigureAwait(false); } - private bool IsKnownPendingId(string callId) => - this._pendingNonApprovalCallIds.ContainsKey(callId) || this._approvalSnapshots.ContainsKey(callId); + private IEnumerable GetSideEffectMessages(ExternalInputResponse response) + { + if (!this._hasApprovalRequiredInvocation) + { + return response.Messages; + } + + return this.FilterRejectedApprovalResults(response); + } /// public override ValueTask ResetAsync() { this._approvalSnapshots.Clear(); + this._hasApprovalRequiredInvocation = false; this._pendingNonApprovalCallIds.Clear(); return default; } /// /// - /// Persists pending approval snapshots and non-approval call ids so they survive - /// checkpoint/restore cycles. + /// Persists approval classification, pending approval snapshots, and non-approval + /// call ids so they survive checkpoint/restore cycles. /// protected override async ValueTask OnCheckpointingAsync(IWorkflowContext context, CancellationToken cancellationToken = default) { Dictionary snapshotCopy = this._approvalSnapshots.ToDictionary(kvp => kvp.Key, kvp => kvp.Value, StringComparer.Ordinal); await context.QueueStateUpdateAsync(ApprovalSnapshotStateKey, snapshotCopy, null, cancellationToken).ConfigureAwait(false); + await context.QueueStateUpdateAsync(HasApprovalRequiredInvocationStateKey, this._hasApprovalRequiredInvocation, null, cancellationToken).ConfigureAwait(false); + List pendingCopy = [.. this._pendingNonApprovalCallIds.Keys]; await context.QueueStateUpdateAsync(PendingCallIdsStateKey, pendingCopy, null, cancellationToken).ConfigureAwait(false); @@ -259,7 +296,6 @@ protected override async ValueTask OnCheckpointingAsync(IWorkflowContext context protected override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext context, CancellationToken cancellationToken = default) { await base.OnCheckpointRestoredAsync(context, cancellationToken).ConfigureAwait(false); - this._approvalSnapshots.Clear(); Dictionary? snapshots = await context.ReadStateAsync>( ApprovalSnapshotStateKey, null, cancellationToken).ConfigureAwait(false); @@ -271,6 +307,10 @@ protected override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext co } } + this._hasApprovalRequiredInvocation = + await context.ReadStateAsync(HasApprovalRequiredInvocationStateKey, null, cancellationToken).ConfigureAwait(false) + || !this._approvalSnapshots.IsEmpty; + this._pendingNonApprovalCallIds.Clear(); List? pending = await context.ReadStateAsync>( PendingCallIdsStateKey, null, cancellationToken).ConfigureAwait(false); @@ -289,6 +329,7 @@ protected override async ValueTask OnCheckpointRestoredAsync(IWorkflowContext co if (legacy is not null) { this._approvalSnapshots.TryAdd(this.Id, legacy); + this._hasApprovalRequiredInvocation = true; await context.QueueStateUpdateAsync( LegacyApprovalSnapshotStateKey, null, null, cancellationToken).ConfigureAwait(false); } @@ -341,6 +382,41 @@ private static IEnumerable TransformConversationMessages(IEnumerabl } } + private IEnumerable FilterRejectedApprovalResults(ExternalInputResponse response) + { + foreach (ChatMessage message in response.Messages) + { + List contents = + [.. message.Contents.Where(c => c is not ToolApprovalResponseContent + && (c is not FunctionResultContent functionResult + || this.IsCorrelatedPendingNonApprovalResult(functionResult, response.RequestId)))]; + if (contents.Count == message.Contents.Count) + { + yield return message; + } + else if (contents.Count > 0) + { + yield return new ChatMessage + { + Role = message.Role, + AuthorName = message.AuthorName, + Contents = contents, + MessageId = message.MessageId, + CreatedAt = message.CreatedAt, + RawRepresentation = message.RawRepresentation, + AdditionalProperties = message.AdditionalProperties, + }; + } + } + } + + private bool IsCorrelatedPendingNonApprovalResult(FunctionResultContent functionResult, string? responseRequestId) => + this._pendingNonApprovalCallIds.ContainsKey(functionResult.CallId) + && IsCorrelatedResponse(functionResult.CallId, responseRequestId); + + private static bool IsCorrelatedResponse(string contentId, string? responseRequestId) => + responseRequestId is null || string.Equals(contentId, responseRequestId, StringComparison.Ordinal); + private async ValueTask AssignResultAsync(IWorkflowContext context, FunctionResultContent result) { if (this.Model.Output?.Result is null) diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net10.0/PublicAPI.Unshipped.txt index ab058de62d4..a36677538eb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.Agents.AI.Workflows.Declarative.Events.ExternalInputResponse.RequestId.get -> string? diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net472/PublicAPI.Unshipped.txt index ab058de62d4..a36677538eb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.Agents.AI.Workflows.Declarative.Events.ExternalInputResponse.RequestId.get -> string? diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net8.0/PublicAPI.Unshipped.txt index ab058de62d4..a36677538eb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.Agents.AI.Workflows.Declarative.Events.ExternalInputResponse.RequestId.get -> string? diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net9.0/PublicAPI.Unshipped.txt index ab058de62d4..a36677538eb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.Agents.AI.Workflows.Declarative.Events.ExternalInputResponse.RequestId.get -> string? diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index ab058de62d4..a36677538eb 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI.Workflows.Declarative/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -1 +1,2 @@ #nullable enable +Microsoft.Agents.AI.Workflows.Declarative.Events.ExternalInputResponse.RequestId.get -> string? diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/ExternalRequest.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/ExternalRequest.cs index e1c1765349c..419c4a50f29 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/ExternalRequest.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/ExternalRequest.cs @@ -77,7 +77,8 @@ public static ExternalRequest Create(RequestPort port, [NotNull] object data, st /// Thrown when the input data object does not match the expected response type. public ExternalResponse CreateResponse(object data) { - if (!Throw.IfNull(this.PortInfo).ResponseType.IsMatchPolymorphic(Throw.IfNull(data).GetType())) + data = Throw.IfNull(data); + if (!Throw.IfNull(this.PortInfo).ResponseType.IsMatchPolymorphic(data.GetType())) { throw new InvalidOperationException( $"Message type {data.GetType().Name} does not match expected response type {this.PortInfo.ResponseType.TypeName} of input port {this.PortInfo.PortId}."); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/InProc/InProcessRunnerContext.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/InProc/InProcessRunnerContext.cs index 8ccfb957294..353c38f25c9 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/InProc/InProcessRunnerContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/InProc/InProcessRunnerContext.cs @@ -167,6 +167,8 @@ async ValueTask PrepareExternalDeliveryAsync() $"Response port id '{response.PortInfo.PortId}' does not match the originating port id for request {response.RequestId}."); } + response = pendingRequest.RewrapResponse(response); + // Consume only after validation so a rejected response leaves the legitimate one able to complete. if (!this.CompleteRequest(response.RequestId)) { diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/InvokeToolWorkflowTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/InvokeToolWorkflowTest.cs index 00f00307c69..01db8d7d693 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/InvokeToolWorkflowTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.IntegrationTests/InvokeToolWorkflowTest.cs @@ -73,7 +73,7 @@ private async Task RunInvokeFunctionToolTestAsync( string workflowPath = GetWorkflowPath(workflowFileName); IEnumerable functionTools = new MenuPlugin().GetTools(); Dictionary functionMap = functionTools.ToDictionary(tool => tool.Name, tool => tool); - DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(externalConversation: false); + DeclarativeWorkflowOptions workflowOptions = await this.CreateOptionsAsync(externalConversation: false, functionTools); Workflow workflow = DeclarativeWorkflowBuilder.Build(workflowPath, workflowOptions); WorkflowHarness harness = new(workflow, runId: Path.GetFileNameWithoutExtension(workflowPath)); diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowJsonOptionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowJsonOptionsTests.cs index eb3b2c8aad4..e6714ba52e9 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowJsonOptionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/DeclarativeWorkflowJsonOptionsTests.cs @@ -67,11 +67,12 @@ public void ExternalInputRequest_RoundTrip_WithApprovalAndFunctionCallContent() [Fact] public void ExternalInputResponse_RoundTrip() { - ExternalInputResponse copy = RoundTrip(new ExternalInputResponse(new ChatMessage(ChatRole.User, "ok"))); + ExternalInputResponse copy = RoundTrip(new ExternalInputResponse([new ChatMessage(ChatRole.User, "ok")], "request-1")); ChatMessage messageCopy = Assert.Single(copy.Messages); Assert.Equal(ChatRole.User, messageCopy.Role); Assert.Equal("ok", messageCopy.Text); + Assert.Equal("request-1", copy.RequestId); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Events/ExternalInputRequestTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Events/ExternalInputRequestTest.cs index d732c11099f..3dc68a8869c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Events/ExternalInputRequestTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/Events/ExternalInputRequestTest.cs @@ -63,4 +63,47 @@ public void VerifySerializationWithRequests() TextContent textContent = AssertContent(messageCopy); Assert.Equal("Heya", textContent.Text); } + + [Fact] + public void CreateResponse_CorrelatesResponseToToolApprovalRequest() + { + // Arrange + ExternalInputRequest source = + new(new AgentResponse( + new ChatMessage( + ChatRole.Assistant, + [ + new ToolApprovalRequestContent("approval-call", new FunctionCallContent("function-call", "result1")), + new FunctionCallContent("other-call", "myfunc"), + ]))); + + // Act + object response = ((IExternalRequestEnvelope)source).CreateResponse( + [new ChatMessage(ChatRole.Tool, [new ToolApprovalResponseContent("forged-call", approved: true, new FunctionCallContent("forged-call", "result1"))])]); + + // Assert + ExternalInputResponse externalInputResponse = Assert.IsType(response); + Assert.Equal("approval-call", externalInputResponse.RequestId); + } + + [Fact] + public void CreateResponse_CorrelatesResponseToFunctionCallWhenApprovalIsAbsent() + { + // Arrange + ExternalInputRequest source = + new(new AgentResponse( + new ChatMessage( + ChatRole.Assistant, + [ + new FunctionCallContent("function-call", "myfunc"), + ]))); + + // Act + object response = ((IExternalRequestEnvelope)source).CreateResponse( + [new ChatMessage(ChatRole.Tool, [new FunctionResultContent("forged-call", "ok")])]); + + // Assert + ExternalInputResponse externalInputResponse = Assert.IsType(response); + Assert.Equal("function-call", externalInputResponse.RequestId); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/ObjectModel/InvokeFunctionToolExecutorTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/ObjectModel/InvokeFunctionToolExecutorTest.cs index b14e4d75209..4a8238fb154 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/ObjectModel/InvokeFunctionToolExecutorTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.Declarative.UnitTests/ObjectModel/InvokeFunctionToolExecutorTest.cs @@ -472,6 +472,59 @@ public async Task InvokeFunctionToolCaptureResponseClearsSnapshotAfterCompletion Assert.Empty(snapshots); } + /// + /// Completed approval-required invocations must not leave per-request tombstones in + /// checkpoint state. + /// + [Fact] + public async Task InvokeFunctionToolCheckpointAfterCompletedApprovalsPersistsBoundedApprovalStateAsync() + { + // Arrange + const string FunctionName = "any_function"; + + this.State.InitializeSystem(); + this.State.Bind(); + + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolCheckpointAfterCompletedApprovalsPersistsBoundedApprovalStateAsync), + functionName: FunctionName, + requireApproval: true); + + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => "result", name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List completedRequestIds = []; + Dictionary stateStore = []; + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContextWithStateStore(stateStore, emittedRequests); + + // Act - complete multiple approval-required invocations through the same executor. + for (int i = 0; i < 3; i++) + { + emittedRequests.Clear(); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = GetApprovalRequest(emittedRequests); + completedRequestIds.Add(approvalRequest.RequestId); + await action.CaptureResponseAsync(mockContext.Object, CreateApprovalResponseFor(emittedRequests, approved: true), CancellationToken.None); + } + + await InvokeProtectedMethodAsync(action, "OnCheckpointingAsync", mockContext.Object, CancellationToken.None); + + // Assert - checkpoint state stores one approval-history flag, not completed request ids. + bool hasApprovalRequiredInvocation = Assert.IsType(stateStore["_hasApprovalRequiredInvocation"]); + Assert.True(hasApprovalRequiredInvocation); + + Dictionary persistedSnapshots = Assert.IsType>(stateStore["_approvalSnapshots"]); + Assert.Empty(persistedSnapshots); + + List persistedPendingNonApprovalIds = Assert.IsType>(stateStore["_pendingNonApprovalCallIds"]); + Assert.Empty(persistedPendingNonApprovalIds); + + Assert.DoesNotContain(stateStore.Values.OfType>(), persistedIds => completedRequestIds.Any(persistedIds.Contains)); + } + /// /// Each ExecuteAsync invocation must produce a unique per-invocation request id on /// both the FunctionCallContent.CallId and the ToolApprovalRequestContent.RequestId. @@ -643,6 +696,119 @@ public async Task InvokeFunctionToolNonApprovalCallIdsAreDistinctPerInvocationAs Assert.NotEqual(action.Id, fcc2.CallId); } + /// + /// A non-approval function result must not satisfy a concurrent approval-required + /// invocation just because its call id is pending on the same executor. + /// + [Fact] + public async Task InvokeFunctionToolApprovalResponseRejectsConcurrentNonApprovalResultAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string RequireApprovalVariable = "NeedsApproval"; + const string ResultVariable = "Result"; + const string CrossModeResult = "cross-mode-result"; + + this.State.Set(RequireApprovalVariable, FormulaValue.New(false)); + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModelWithVariableRequireApproval( + displayName: nameof(InvokeFunctionToolApprovalResponseRejectsConcurrentNonApprovalResultAsync), + functionName: FunctionName, + requireApprovalVariableName: RequireApprovalVariable, + outputResultVariable: ResultVariable); + + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => "registered-result", name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + string nonApprovalCallId = GetFunctionCall(emittedRequests[0]).CallId; + + this.State.Set(RequireApprovalVariable, FormulaValue.New(true)); + this.State.Bind(); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + string approvalRequestId = GetApprovalRequest(emittedRequests[1]).RequestId; + + ExternalInputResponse crossModeResponse = CreateFunctionResultResponse( + request: emittedRequests[1], + expectedRequestId: approvalRequestId, + callId: nonApprovalCallId, + result: CrossModeResult); + + // Act + await action.CaptureResponseAsync(mockContext.Object, crossModeResponse, CancellationToken.None); + + // Assert + Assert.DoesNotContain(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == CrossModeResult); + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value.Contains("No pending approval")); + } + + /// + /// A non-approval result remains valid while an approval-required invocation is pending + /// when the response is correlated to the non-approval request that emitted it. + /// + [Fact] + public async Task InvokeFunctionToolConcurrentNonApprovalResultIsAcceptedForItsOwnRequestAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string RequireApprovalVariable = "NeedsApproval"; + const string ResultVariable = "Result"; + const string NonApprovalResult = "non-approval-result"; + + this.State.Set(RequireApprovalVariable, FormulaValue.New(false)); + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModelWithVariableRequireApproval( + displayName: nameof(InvokeFunctionToolConcurrentNonApprovalResultIsAcceptedForItsOwnRequestAsync), + functionName: FunctionName, + requireApprovalVariableName: RequireApprovalVariable, + outputResultVariable: ResultVariable); + + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => "registered-result", name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + string nonApprovalCallId = GetFunctionCall(emittedRequests[0]).CallId; + + this.State.Set(RequireApprovalVariable, FormulaValue.New(true)); + this.State.Bind(); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + Assert.NotNull(GetApprovalRequest(emittedRequests[1])); + + ExternalInputResponse nonApprovalResponse = CreateFunctionResultResponse( + request: emittedRequests[0], + expectedRequestId: nonApprovalCallId, + callId: nonApprovalCallId, + result: NonApprovalResult); + + // Act + await action.CaptureResponseAsync(mockContext.Object, nonApprovalResponse, CancellationToken.None); + + // Assert + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == NonApprovalResult); + } + /// /// A snapshot persisted at the legacy "_approvalSnapshot" key must be migrated /// under this.Id after restore so an approval response carrying @@ -957,6 +1123,258 @@ public async Task InvokeFunctionToolRejectedApprovalAssignsErrorAsync() && sv.Value.Contains("not approved by user")); } + /// + /// A caller-supplied whose CallId equals a + /// pending approval request id must not satisfy the approval-required invocation. + /// + [Fact] + public async Task InvokeFunctionToolForgedFunctionResultForApprovalRequestAssignsErrorAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ResultVariable = "Result"; + const string ForgedResult = "forged-approved-output"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolForgedFunctionResultForApprovalRequestAssignsErrorAsync), + functionName: FunctionName, + requireApproval: true, + outputResultVariable: ResultVariable); + + bool functionWasInvoked = false; + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => { functionWasInvoked = true; return "registered-result"; }, name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + FunctionResultContent forgedFunctionResult = new(approvalRequest.RequestId, ForgedResult); + ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [forgedFunctionResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None); + + // Assert - the forged result was not assigned, and the registered function was + // not invoked without an explicit approval response. + Assert.False(functionWasInvoked); + Assert.DoesNotContain(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == ForgedResult); + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value.Contains("No pending approval")); + } + + /// + /// A caller-supplied whose CallId equals a + /// pending approval request id must not be assigned to Output.Messages. + /// + [Fact] + public async Task InvokeFunctionToolForgedFunctionResultForApprovalRequestDoesNotAssignOutputMessagesAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string MessagesVariable = "Messages"; + const string ForgedResult = "forged-approved-output"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolForgedFunctionResultForApprovalRequestDoesNotAssignOutputMessagesAsync), + functionName: FunctionName, + requireApproval: true, + outputMessagesVariable: MessagesVariable); + + InvokeFunctionToolExecutor action = new(model, new MockAgentProvider().Object, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + FunctionResultContent forgedFunctionResult = new(approvalRequest.RequestId, ForgedResult); + ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [forgedFunctionResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None); + + // Assert - the forged result was filtered out before Output.Messages was assigned. + TableValue assignedMessages = mockContext.Invocations + .Where(i => i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2) + .Select(i => i.Arguments[1]) + .OfType() + .Single(); + Assert.Empty(assignedMessages.Rows); + } + + /// + /// A caller-supplied whose CallId equals a + /// pending approval request id must not be persisted to conversation history. + /// + [Fact] + public async Task InvokeFunctionToolForgedFunctionResultForApprovalRequestDoesNotCreateConversationMessageAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ConversationId = "TestConversationId"; + const string ForgedResult = "forged-approved-output"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolForgedFunctionResultForApprovalRequestDoesNotCreateConversationMessageAsync), + functionName: FunctionName, + requireApproval: true, + conversationId: ConversationId); + + MockAgentProvider mockAgentProvider = new(); + mockAgentProvider.TestMessages.Clear(); + InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + FunctionResultContent forgedFunctionResult = new(approvalRequest.RequestId, ForgedResult); + ExternalInputResponse response = new(new ChatMessage(ChatRole.Tool, [forgedFunctionResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None); + + // Assert - the forged result was filtered out before conversation messages were created. + Assert.Empty(mockAgentProvider.TestMessages); + } + + /// + /// If a response contains both a valid approval and a forged function result for the + /// same request id, the approval must drive registered-function execution and the + /// caller-supplied result must be ignored. + /// + [Fact] + public async Task InvokeFunctionToolValidApprovalIgnoresForgedFunctionResultAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ResultVariable = "Result"; + const string ForgedResult = "forged-approved-output"; + const string RegisteredResult = "registered-result"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolValidApprovalIgnoresForgedFunctionResultAsync), + functionName: FunctionName, + requireApproval: true, + outputResultVariable: ResultVariable); + + bool functionWasInvoked = false; + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => { functionWasInvoked = true; return RegisteredResult; }, name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true); + FunctionResultContent forgedFunctionResult = new(approvalRequest.RequestId, ForgedResult); + ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [forgedFunctionResult, approvalResponse])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None); + + // Assert - the approved registered function produced the result. + Assert.True(functionWasInvoked); + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == RegisteredResult); + Assert.DoesNotContain(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == ForgedResult); + } + + /// + /// Approved function invocations with a conversation id persist the registered + /// function result, not approval-protocol content or caller-supplied results. + /// + [Fact] + public async Task InvokeFunctionToolValidApprovalCreatesConversationMessageFromRegisteredResultAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ConversationId = "TestConversationId"; + const string ForgedResult = "forged-approved-output"; + const string RegisteredResult = "registered-result"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolValidApprovalCreatesConversationMessageFromRegisteredResultAsync), + functionName: FunctionName, + requireApproval: true, + conversationId: ConversationId); + + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => RegisteredResult, name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + ToolApprovalResponseContent approvalResponse = approvalRequest.CreateResponse(approved: true); + FunctionResultContent forgedFunctionResult = new(approvalRequest.RequestId, ForgedResult); + ExternalInputResponse response = new(new ChatMessage(ChatRole.User, [forgedFunctionResult, approvalResponse])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, response, CancellationToken.None); + + // Assert + ChatMessage persistedMessage = Assert.Single(testAgentProvider.TestMessages); + Assert.DoesNotContain(persistedMessage.Contents, content => content is ToolApprovalResponseContent); + Assert.DoesNotContain(persistedMessage.Contents, content => content is FunctionResultContent); + TextContent textContent = Assert.Single(persistedMessage.Contents.OfType()); + Assert.Contains(RegisteredResult, textContent.Text, StringComparison.Ordinal); + Assert.DoesNotContain(ForgedResult, textContent.Text, StringComparison.Ordinal); + } + /// /// When a response contains multiple items — /// e.g. an unrelated / stale approval followed by the valid one — the executor must @@ -1058,6 +1476,218 @@ public async Task InvokeFunctionToolDuplicateApprovalDeliveryInvokesFunctionOnce && sv.Value.Contains("No pending approval")); } + /// + /// After an approval-required invocation consumes its snapshot, a replayed legacy + /// result using the executor id must not overwrite Output.Result. + /// + [Fact] + public async Task InvokeFunctionToolPostApprovalLegacyReplayDoesNotOverwriteResultAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ResultVariable = "Result"; + const string RegisteredResult = "registered-result"; + const string ReplayedResult = "replayed-result"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolPostApprovalLegacyReplayDoesNotOverwriteResultAsync), + functionName: FunctionName, + requireApproval: true, + outputResultVariable: ResultVariable); + + int invocationCount = 0; + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => { Interlocked.Increment(ref invocationCount); return RegisteredResult; }, name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ExternalInputResponse approvalResponse = CreateApprovalResponseFor(emittedRequests, approved: true); + FunctionResultContent replayedLegacyResult = new(action.Id, ReplayedResult); + ExternalInputResponse replayResponse = new(new ChatMessage(ChatRole.Tool, [replayedLegacyResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, approvalResponse, CancellationToken.None); + await action.CaptureResponseAsync(mockContext.Object, replayResponse, CancellationToken.None); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == RegisteredResult); + Assert.DoesNotContain(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == ReplayedResult); + } + + /// + /// After an approval-required invocation consumes its snapshot, a replayed legacy + /// result using the executor id must not be assigned to Output.Messages or + /// persisted to conversation history. + /// + [Fact] + public async Task InvokeFunctionToolPostApprovalLegacyReplayDoesNotAssignOutputMessagesOrCreateConversationMessageAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ConversationId = "TestConversationId"; + const string MessagesVariable = "Messages"; + const string ReplayedResult = "replayed-result"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolPostApprovalLegacyReplayDoesNotAssignOutputMessagesOrCreateConversationMessageAsync), + functionName: FunctionName, + requireApproval: true, + conversationId: ConversationId, + outputMessagesVariable: MessagesVariable); + + MockAgentProvider mockAgentProvider = new(); + InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ExternalInputResponse approvalResponse = CreateApprovalResponseFor(emittedRequests, approved: true); + FunctionResultContent replayedLegacyResult = new(action.Id, ReplayedResult); + ExternalInputResponse replayResponse = new(new ChatMessage(ChatRole.Tool, [replayedLegacyResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, approvalResponse, CancellationToken.None); + mockContext.Invocations.Clear(); + mockAgentProvider.TestMessages.Clear(); + await action.CaptureResponseAsync(mockContext.Object, replayResponse, CancellationToken.None); + + // Assert + TableValue assignedMessages = mockContext.Invocations + .Where(i => i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2) + .Select(i => i.Arguments[1]) + .OfType() + .Single(); + Assert.Empty(assignedMessages.Rows); + Assert.Empty(mockAgentProvider.TestMessages); + } + + /// + /// After an approval-required invocation is rejected and consumes its snapshot, a replayed + /// result using the approval request id must not be assigned to Output.Messages or + /// persisted to conversation history. + /// + [Fact] + public async Task InvokeFunctionToolPostRejectionReplayDoesNotAssignOutputMessagesOrCreateConversationMessageAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string ConversationId = "TestConversationId"; + const string MessagesVariable = "Messages"; + const string ReplayedResult = "replayed-result"; + + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModel( + displayName: nameof(InvokeFunctionToolPostRejectionReplayDoesNotAssignOutputMessagesOrCreateConversationMessageAsync), + functionName: FunctionName, + requireApproval: true, + conversationId: ConversationId, + outputMessagesVariable: MessagesVariable); + + MockAgentProvider mockAgentProvider = new(); + InvokeFunctionToolExecutor action = new(model, mockAgentProvider.Object, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + ToolApprovalRequestContent approvalRequest = GetApprovalRequest(emittedRequests); + ExternalInputResponse rejectionResponse = CreateApprovalResponseFor(emittedRequests, approved: false); + FunctionResultContent replayedApprovalResult = new(approvalRequest.RequestId, ReplayedResult); + ExternalInputResponse replayResponse = new(new ChatMessage(ChatRole.Tool, [replayedApprovalResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, rejectionResponse, CancellationToken.None); + mockContext.Invocations.Clear(); + mockAgentProvider.TestMessages.Clear(); + await action.CaptureResponseAsync(mockContext.Object, replayResponse, CancellationToken.None); + + // Assert + TableValue assignedMessages = mockContext.Invocations + .Where(i => i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2) + .Select(i => i.Arguments[1]) + .OfType() + .Single(); + Assert.Empty(assignedMessages.Rows); + Assert.Empty(mockAgentProvider.TestMessages); + } + + /// + /// The approval mode captured at emit time must gate legacy replay matching even if + /// the mutable RequireApproval expression evaluates to false after approval. + /// + [Fact] + public async Task InvokeFunctionToolPostApprovalLegacyReplayUsesEmittedApprovalModeAsync() + { + // Arrange + const string FunctionName = "any_function"; + const string RequireApprovalVariable = "NeedsApproval"; + const string ResultVariable = "Result"; + const string RegisteredResult = "registered-result"; + const string ReplayedResult = "replayed-result"; + + this.State.Set(RequireApprovalVariable, FormulaValue.New(true)); + this.State.InitializeSystem(); + this.State.Bind(); + InvokeFunctionTool model = this.CreateModelWithVariableRequireApproval( + displayName: nameof(InvokeFunctionToolPostApprovalLegacyReplayUsesEmittedApprovalModeAsync), + functionName: FunctionName, + requireApprovalVariableName: RequireApprovalVariable, + outputResultVariable: ResultVariable); + + int invocationCount = 0; + TestFunctionAgentProvider testAgentProvider = new( + [AIFunctionFactory.Create(() => { Interlocked.Increment(ref invocationCount); return RegisteredResult; }, name: FunctionName)]); + InvokeFunctionToolExecutor action = new(model, testAgentProvider, this.State); + + List emittedRequests = []; + Mock mockContext = CreateMockWorkflowContext(emittedRequests); + await action.HandleAsync(new ActionExecutorResult(action.Id), mockContext.Object, CancellationToken.None); + + this.State.Set(RequireApprovalVariable, FormulaValue.New(false)); + this.State.Bind(); + + ExternalInputResponse approvalResponse = CreateApprovalResponseFor(emittedRequests, approved: true); + FunctionResultContent replayedLegacyResult = new(action.Id, ReplayedResult); + ExternalInputResponse replayResponse = new(new ChatMessage(ChatRole.Tool, [replayedLegacyResult])); + + // Act + await action.CaptureResponseAsync(mockContext.Object, approvalResponse, CancellationToken.None); + await action.CaptureResponseAsync(mockContext.Object, replayResponse, CancellationToken.None); + + // Assert + Assert.Equal(1, invocationCount); + Assert.Contains(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == RegisteredResult); + Assert.DoesNotContain(mockContext.Invocations, i => + i.Method.Name == nameof(IWorkflowContext.QueueStateUpdateAsync) + && i.Arguments.Count >= 2 + && i.Arguments[1] is StringValue sv + && sv.Value == ReplayedResult); + } + /// /// A non-approval FunctionResultContent whose CallId equals this.Id is /// consumed and assigned to Output.Result when no pendings are tracked. @@ -1170,6 +1800,45 @@ private static ExternalInputResponse CreateApprovalResponseFor(IReadOnlyList emittedRequests) + { + return Assert.Single(emittedRequests) + .AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + } + + private static ToolApprovalRequestContent GetApprovalRequest(ExternalInputRequest emittedRequest) + { + return emittedRequest.AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + } + + private static FunctionCallContent GetFunctionCall(ExternalInputRequest emittedRequest) + { + return emittedRequest.AgentResponse.Messages + .SelectMany(m => m.Contents) + .OfType() + .Single(); + } + + private static ExternalInputResponse CreateFunctionResultResponse( + ExternalInputRequest request, + string expectedRequestId, + string callId, + string result) + { + object response = ((IExternalRequestEnvelope)request).CreateResponse( + [new ChatMessage(ChatRole.Tool, [new FunctionResultContent(callId, result)])]); + + ExternalInputResponse correlatedResponse = Assert.IsType(response); + Assert.Equal(expectedRequestId, correlatedResponse.RequestId); + return correlatedResponse; + } + private static Mock CreateMockWorkflowContext(List? emittedRequests = null) { Mock mockContext = new(); @@ -1208,6 +1877,9 @@ private static Mock CreateMockWorkflowContextWithStateStore( mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) .Callback, string?, CancellationToken>((key, value, _, _) => stateStore[key] = value) .Returns(default(ValueTask)); + mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback((key, value, _, _) => stateStore[key] = value) + .Returns(default(ValueTask)); mockContext.Setup(c => c.QueueStateUpdateAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Callback((key, value, _, _) => { @@ -1236,6 +1908,9 @@ private static Mock CreateMockWorkflowContextWithStateStore( mockContext.Setup(c => c.ReadStateAsync>(It.IsAny(), It.IsAny(), It.IsAny())) .Returns((key, _, _) => new ValueTask?>(stateStore.TryGetValue(key, out object? val) ? val as List : null)); + mockContext.Setup(c => c.ReadStateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((key, _, _) => + new ValueTask(stateStore.TryGetValue(key, out object? val) && val is bool boolValue && boolValue)); mockContext.Setup(c => c.ReadStateAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns((key, _, _) => new ValueTask(stateStore.TryGetValue(key, out object? val) ? val as ApprovalSnapshot : null)); @@ -1265,6 +1940,8 @@ private sealed class TestFunctionAgentProvider : ResponseAgentProvider private readonly Action? _onInvoke; private readonly Action? _onInvokeArguments; + public List TestMessages { get; } = []; + public TestFunctionAgentProvider( IEnumerable functions, Action? onInvoke = null, @@ -1287,8 +1964,11 @@ internal void RecordInvocation(string name, AIFunctionArguments? arguments) public override Task CreateConversationAsync(CancellationToken cancellationToken = default) => throw new NotSupportedException(); - public override Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default) => - throw new NotSupportedException(); + public override Task CreateMessageAsync(string conversationId, ChatMessage conversationMessage, CancellationToken cancellationToken = default) + { + this.TestMessages.Add(conversationMessage); + return Task.FromResult(conversationMessage); + } public override Task GetMessageAsync(string conversationId, string messageId, CancellationToken cancellationToken = default) => throw new NotSupportedException(); @@ -1355,7 +2035,8 @@ private InvokeFunctionTool CreateModel( string? conversationId = null, string? argumentKey = null, string? argumentValue = null, - string? outputResultVariable = null) + string? outputResultVariable = null, + string? outputMessagesVariable = null) { InvokeFunctionTool.Builder builder = new() { @@ -1375,11 +2056,12 @@ private InvokeFunctionTool CreateModel( builder.Arguments.Add(argumentKey, ValueExpression.Literal(new StringDataValue(argumentValue))); } - if (outputResultVariable is not null) + if (outputResultVariable is not null || outputMessagesVariable is not null) { builder.Output = new InvokeToolOutput.Builder { - Result = new InitializablePropertyPath(PropertyPath.TopicVariable(outputResultVariable), isInitializer: false), + Result = outputResultVariable is not null ? new InitializablePropertyPath(PropertyPath.TopicVariable(outputResultVariable), isInitializer: false) : null, + Messages = outputMessagesVariable is not null ? new InitializablePropertyPath(PropertyPath.TopicVariable(outputMessagesVariable), isInitializer: false) : null, }; } @@ -1414,5 +2096,23 @@ private InvokeFunctionTool CreateModelWithVariableArgument( return AssignParent(builder); } + private InvokeFunctionTool CreateModelWithVariableRequireApproval( + string displayName, string functionName, string requireApprovalVariableName, string outputResultVariable) + { + InvokeFunctionTool.Builder builder = new() + { + Id = this.CreateActionId(), + DisplayName = this.FormatDisplayName(displayName), + FunctionName = new StringExpression.Builder(StringExpression.Literal(functionName)), + RequireApproval = new BoolExpression.Builder( + BoolExpression.Variable(PropertyPath.TopicVariable(requireApprovalVariableName))), + Output = new InvokeToolOutput.Builder + { + Result = new InitializablePropertyPath(PropertyPath.TopicVariable(outputResultVariable), isInitializer: false), + }, + }; + return AssignParent(builder); + } + #endregion } diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/SampleSmokeTest.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/SampleSmokeTest.cs index fa1ce7b974f..eb4e9589a55 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/SampleSmokeTest.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/SampleSmokeTest.cs @@ -117,17 +117,7 @@ internal async Task Test_RunSample_Step5Async(ExecutionEnvironment environment) { using StringWriter writer = new(); - VerifyingPlaybackResponder responder = new( - // Iteration 1 - ("Guess the number.", 50), - ("Your guess was too high. Try again.", 23), - - // Iteration 2 - ("Your guess was too high. Try again.", 23), - ("Your guess was too low. Try again.", 42) - ); - - string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: responder.InvokeNext, environment.ToWorkflowExecutionEnvironment()); + string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: GetGuessResponse, environment.ToWorkflowExecutionEnvironment()); Assert.Equal("You guessed correctly! You Win!", guessResult); } @@ -139,17 +129,7 @@ internal async Task Test_RunSample_Step5aAsync(ExecutionEnvironment environment) { using StringWriter writer = new(); - VerifyingPlaybackResponder responder = new( - // Iteration 1 - ("Guess the number.", 50), - ("Your guess was too high. Try again.", 23), - - // Iteration 2 - ("Your guess was too high. Try again.", 23), - ("Your guess was too low. Try again.", 42) - ); - - string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: responder.InvokeNext, environment.ToWorkflowExecutionEnvironment(), rehydrateToRestore: true); + string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: GetGuessResponse, environment.ToWorkflowExecutionEnvironment(), rehydrateToRestore: true); Assert.Equal("You guessed correctly! You Win!", guessResult); } @@ -161,21 +141,11 @@ internal async Task Test_RunSample_Step5bAsync(ExecutionEnvironment environment) { using StringWriter writer = new(); - VerifyingPlaybackResponder responder = new( - // Iteration 1 - ("Guess the number.", 50), - ("Your guess was too high. Try again.", 23), - - // Iteration 2 - ("Your guess was too high. Try again.", 23), - ("Your guess was too low. Try again.", 42) - ); - JsonSerializerOptions options = new(SampleJsonContext.Default.Options); options.MakeReadOnly(); CheckpointManager memoryJsonManager = CheckpointManager.CreateJson(new InMemoryJsonStore(), options); - string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: responder.InvokeNext, environment.ToWorkflowExecutionEnvironment(), rehydrateToRestore: true, checkpointManager: memoryJsonManager); + string guessResult = await Step5EntryPoint.RunAsync(writer, userGuessCallback: GetGuessResponse, environment.ToWorkflowExecutionEnvironment(), rehydrateToRestore: true, checkpointManager: memoryJsonManager); Assert.Equal("You guessed correctly! You Win!", guessResult); } @@ -555,6 +525,15 @@ internal async Task Test_RunSample_Step14a_SharedState_IsolatedAcrossSubworkflow Assert.IsType(actualError); } + + private static int GetGuessResponse(string prompt) => + prompt switch + { + "Guess the number." => 50, + "Your guess was too high. Try again." => 23, + "Your guess was too low. Try again." => 42, + _ => throw new InvalidOperationException($"Unexpected guess prompt: {prompt}") + }; } internal sealed class VerifyingPlaybackResponder diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestExternalEnvelopes.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestExternalEnvelopes.cs new file mode 100644 index 00000000000..cae9a7f6594 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestExternalEnvelopes.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.Workflows.UnitTests; + +internal sealed class TestExternalRequestEnvelope : IExternalRequestEnvelope +{ + [JsonConstructor] + public TestExternalRequestEnvelope(FunctionCallContent functionCall) + { + this.FunctionCall = functionCall; + } + + public FunctionCallContent FunctionCall { get; } + + AIContent? IExternalRequestEnvelope.GetInnerRequestContent() => this.FunctionCall; + + object IExternalRequestEnvelope.CreateResponse(IList messages) => messages; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestJsonContext.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestJsonContext.cs index aaba9423d7d..db8e8eda3a8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestJsonContext.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/TestJsonContext.cs @@ -7,5 +7,6 @@ namespace Microsoft.Agents.AI.Workflows.UnitTests; // Checkpointing Types [JsonSerializable(typeof(TestJsonSerializable))] +[JsonSerializable(typeof(TestExternalRequestEnvelope))] [ExcludeFromCodeCoverage] internal sealed partial class TestJsonContext : JsonSerializerContext;