[26.x] [AI Test Tool] Backport changes

src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al

@@ -25,6 +25,8 @@ permissionset 149031 "AI Test Toolkit - Obj"
         codeunit "AIT Run History" = X,
         xmlport "AIT Test Suite Import/Export" = X,
         page "AIT CommandLine Card" = X,
+        page "AIT Column Mappings" = X,
+        page "AIT Evaluators" = X,
         page "AIT Test Data" = X,
         page "AIT Test Data Compare" = X,
         page "AIT Batch Run Dialog" = X,

src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al

@@ -94,6 +94,60 @@ codeunit 149044 "AIT Test Context"
         AITTestContextImpl.SetAnswerForQnAEvaluation(Answer);
     end;
 
+    /// <summary>
+    /// Sets the query and respone for a single-turn evaluation.
+    /// Optionally, a context can be provided.
+    /// </summary>
+    /// <param name="Query">The query as text.</param>
+    /// <param name="Response">The response as text.</param>
+    /// <param name="Context">The context as text.</param>
+    procedure SetQueryResponse(Query: Text; Response: Text; Context: Text)
+    begin
+        AITTestContextImpl.SetQueryResponse(Query, Response, Context);
+    end;
+
+    /// <summary>
+    /// Sets the query and response for a single-turn evaluation.
+    /// </summary>
+    /// <param name="Query">The query as text.</param>
+    /// <param name="Response">The response as text.</param>
+    procedure SetQueryResponse(Query: Text; Response: Text)
+    begin
+        AITTestContextImpl.SetQueryResponse(Query, Response, '');
+    end;
+
+    /// <summary>
+    /// Adds a message to the current test iteration.
+    /// This is used for multi-turn tests to add messages to the output.
+    /// </summary>
+    /// <param name="Content">The content of the message.</param>
+    /// <param name="Role">The role of the message (e.g., 'user', 'assistant').</param>
+    /// <param name="Context">The context of the message.</param>
+    procedure AddMessage(Content: Text; Role: Text; Context: Text)
+    begin
+        AITTestContextImpl.AddMessage(Content, Role, Context);
+    end;
+
+    /// <summary>
+    /// Adds a message to the current test iteration.
+    /// This is used for multi-turn tests to add messages to the output.
+    /// </summary>
+    /// <param name="Content">The content of the message.</param>
+    /// <param name="Role">The role of the message (e.g., 'user', 'assistant').</param>
+    procedure AddMessage(Content: Text; Role: Text)
+    begin
+        AITTestContextImpl.AddMessage(Content, Role, '');
+    end;
+
+    /// <summary>
+    /// Sets the test output for the current iteration.
+    /// </summary>
+    /// <param name="TestOutputJson">The test output.</param>
+    procedure SetTestOutput(TestOutputJson: Codeunit "Test Output Json")
+    begin
+        AITTestContextImpl.SetTestOutput(TestOutputJson);
+    end;
+
     /// <summary>
     /// Sets the test output for the current iteration.
     /// </summary>
@@ -123,6 +177,35 @@ codeunit 149044 "AIT Test Context"
         AITTestContextImpl.SetTestMetric(TestMetric);
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(Accuracy: Decimal)
+    begin
+        AITTestContextImpl.SetAccuracy(Accuracy);
+    end;
+
+    /// <summary>
+    /// Gets the AITTestSuite associated with the run.
+    /// </summary>
+    /// <param name="AITTestSuite">AITTestSuite associated with the run.</param>
+    procedure GetAITTestSuite(var AITTestSuite: Record "AIT Test Suite")
+    begin
+        AITTestContextImpl.GetAITTestSuite(AITTestSuite);
+    end;
+
+    /// <summary>
+    /// Integration event that is raised after a test run is completed.
+    /// </summary>
+    /// <param name="Code">The code of the test run.</param>
+    /// <param name="Version">The version of the test run.</param>
+    /// <param name="Tag">The tag of the test run.</param>
+    [IntegrationEvent(false, false)]
+    internal procedure OnAfterRunComplete(Code: Code[10]; Version: Integer; Tag: Text[20])
+    begin
+    end;
+
     var
         AITTestContextImpl: Codeunit "AIT Test Context Impl.";
 }

src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al

@@ -18,9 +18,13 @@ codeunit 149043 "AIT Test Context Impl."
     var
         AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
         GlobalTestOutputJson: Codeunit "Test Output Json";
+        GlobalAccuracy: Decimal;
         CurrentTurn: Integer;
         NumberOfTurns: Integer;
         IsMultiTurn: Boolean;
+        AccuracySetManually: Boolean;
+        AccuracyErr: Label 'Accuracy must be between 0 and 1.';
+        OnlySingleTurnErr: Label 'A query-and-response pair cannot be used in multi-turn tests. Use AddMessage instead.';
         AnswerTok: Label 'answer', Locked = true;
         ContextTok: Label 'context', Locked = true;
         GroundTruthTok: Label 'ground_truth', Locked = true;
@@ -29,6 +33,12 @@ codeunit 149043 "AIT Test Context Impl."
         TestSetupTok: Label 'test_setup', Locked = true;
         QuestionTok: Label 'question', Locked = true;
         TurnsTok: Label 'turns', Locked = true;
+        MessagesTok: Label 'messages', Locked = true;
+        QueryTok: Label 'query', Locked = true;
+        ResponseTok: Label 'response', Locked = true;
+        RoleTok: Label 'role', Locked = true;
+        ContentTok: Label 'content', Locked = true;
+        ConversationTok: Label 'conversation', Locked = true;
 
     /// <summary>
     /// Returns the Test Input value as Test Input Json Codeunit from the input dataset for the current iteration.
@@ -103,6 +113,64 @@ codeunit 149043 "AIT Test Context Impl."
         SetSuiteTestOutput(CurrentTestOutputJson.ToText());
     end;
 
+    /// <summary>
+    /// Sets the query and respone for a single-turn evaluation.
+    /// Optionally, a context can be provided.
+    /// </summary>
+    /// <param name="Query">The query as text.</param>
+    /// <param name="Response">The response as text.</param>
+    /// <param name="Context">The context as text.</param>
+    procedure SetQueryResponse(Query: Text; Response: Text; Context: Text)
+    var
+        AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt";
+        CurrentTestOutputJson: Codeunit "Test Output Json";
+        TestOutputCU: Codeunit "Test Output";
+    begin
+        if IsMultiTurn then
+            Error(OnlySingleTurnErr);
+
+        CurrentTestOutputJson.Initialize();
+        CurrentTestOutputJson.Add(QueryTok, Query);
+        CurrentTestOutputJson.Add(ResponseTok, Response);
+
+        if Context <> '' then
+            CurrentTestOutputJson.Add(ContextTok, Context);
+
+        TestOutputCU.TestData().Initialize(CurrentTestOutputJson.ToText());
+
+        AITTestSuiteMgt.SetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestOutputCU.Testdata().ToText());
+    end;
+
+    /// <summary>
+    /// Adds a message to the current test iteration.
+    /// This is used for multi-turn tests to add messages to the output.
+    /// </summary>
+    /// <param name="Content">The content of the message.</param>
+    /// <param name="Role">The role of the message (e.g., 'user', 'assistant').</param>
+    /// <param name="Context">The context of the message (can be blank).</param>
+    procedure AddMessage(Content: Text; Role: Text; Context: Text)
+    var
+        CurrentTestOutputJson: Codeunit "Test Output Json";
+    begin
+        CurrentTestOutputJson.Initialize();
+        CurrentTestOutputJson.Add(ContentTok, Content);
+        CurrentTestOutputJson.Add(RoleTok, Role);
+
+        if Context <> '' then
+            CurrentTestOutputJson.Add(ContextTok, Context);
+
+        AddMessageToOutput(CurrentTestOutputJson.ToText());
+    end;
+
+    /// <summary>
+    /// Sets the test output for the current iteration.
+    /// </summary>
+    /// <param name="TestOutputJson">The test output.</param>
+    procedure SetTestOutput(TestOutputJson: Codeunit "Test Output Json")
+    begin
+        SetSuiteTestOutput(TestOutputJson.ToText());
+    end;
+
     /// <summary>
     /// Sets the test output for the current iteration.
     /// </summary>
@@ -146,6 +214,34 @@ codeunit 149043 "AIT Test Context Impl."
         SetSuiteTestOutput(CurrentTestOutputJson.ToText());
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(Accuracy: Decimal)
+    begin
+        if (Accuracy < 0) or (Accuracy > 1) then
+            Error(AccuracyErr);
+
+        AccuracySetManually := true;
+        GlobalAccuracy := Accuracy;
+    end;
+
+    /// <summary>
+    /// Gets the accuracy of the test. Can only be retrieved if the accuracy of the test was already set manually.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    /// <returns>True if it was possible to get the accuracy, false otherwise.</returns>
+    procedure GetAccuracy(var Accuracy: Decimal): Boolean
+    begin
+        if AccuracySetManually then begin
+            Accuracy := GlobalAccuracy;
+            exit(true);
+        end;
+
+        exit(false);
+    end;
+
     /// <summary>
     /// Sets to next turn for multiturn testing.
     /// </summary>
@@ -164,14 +260,34 @@ codeunit 149043 "AIT Test Context Impl."
     end;
 
     /// <summary>
-    /// Gets the current turn for multiturn testing. Turns start from turn 0.
+    /// Gets the current turn for multiturn testing. Turns start from turn 1.
     /// </summary>
     /// <returns>The current turn number.</returns>
     procedure GetCurrentTurn(): Integer
     begin
         exit(CurrentTurn);
     end;
 
+    /// <summary>
+    /// Gets the total number of turns for multiturn testing.
+    /// </summary>
+    /// <returns>The total number of turns for the line.</returns>
+    procedure GetNumberOfTurns(): Integer
+    begin
+        exit(NumberOfTurns);
+    end;
+
+    /// <summary>
+    /// Returns the AITTestSuite associated with the run.
+    /// </summary>
+    /// <param name="AITTestSuite">AITTestSuite associated with the run.</param>
+    procedure GetAITTestSuite(var AITTestSuite: Record "AIT Test Suite")
+    var
+        AITTestRunIteration: Codeunit "AIT Test Run Iteration";
+    begin
+        AITTestRunIteration.GetAITTestSuite(AITTestSuite);
+    end;
+
     /// <summary>
     /// This method starts the scope of the Run Procedure scenario.
     /// </summary>
@@ -205,12 +321,16 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
         TurnsInputJson: Codeunit "Test Input Json";
     begin
-        CurrentTurn := 0;
+        AccuracySetManually := false;
+        GlobalAccuracy := 0;
+        CurrentTurn := 1;
         GlobalTestOutputJson.Initialize();
         TurnsInputJson := TestInput.GetTestInput().ElementExists(TurnsTok, IsMultiTurn);
 
         if IsMultiTurn then
-            NumberOfTurns := TurnsInputJson.GetElementCount() - 1;
+            NumberOfTurns := TurnsInputJson.GetElementCount()
+        else
+            NumberOfTurns := 1;
     end;
 
     /// <summary>
@@ -223,11 +343,29 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
     begin
         if IsMultiTurn then
-            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn).Element(ElementName)
+            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn - 1).Element(ElementName)
         else
             TestInputJson := TestInput.GetTestInput(ElementName);
     end;
 
+    /// <summary>
+    /// Adds a message to the test output for the current iteration.
+    /// </summary>
+    local procedure AddMessageToOutput(Output: Text)
+    var
+        AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt";
+        TestOutputCU: Codeunit "Test Output";
+    begin
+        if not TestOutputCU.TestData().ElementExists(ConversationTok) then begin
+            TestOutputCU.TestData().Add(ConversationTok, '');
+            TestOutputCU.TestData().Element(ConversationTok).AddArray(MessagesTok);
+        end;
+
+        TestOutputCU.TestData().Element(ConversationTok).Element(MessagesTok).Add(Output);
+
+        AITTestSuiteMgt.SetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestOutputCU.Testdata().ToText());
+    end;
+
     /// <summary>
     /// Sets the test output for the current iteration.
     /// </summary>

src/Tools/AI Test Toolkit/src/AITTestRunInputHandler.Codeunit.al

@@ -0,0 +1,42 @@
+// ------------------------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+// ------------------------------------------------------------------------------------------------
+
+namespace System.TestTools.AITestToolkit;
+
+using System.TestTools.TestRunner;
+
+codeunit 149045 "AIT Test Run Input Handler"
+{
+    SingleInstance = true;
+    EventSubscriberInstance = Manual;
+    InherentEntitlements = X;
+    InherentPermissions = X;
+
+    var
+        TestInputGroupCode: Code[100];
+        TestInputCode: Code[100];
+
+    internal procedure SetInput(InputGroupCode: Code[100]; InputCode: Code[100])
+    begin
+        TestInputGroupCode := InputGroupCode;
+        TestInputCode := InputCode;
+    end;
+
+    [EventSubscriber(ObjectType::Codeunit, Codeunit::"AIT Test Run Iteration", 'OnBeforeRunIteration', '', false, false)]
+    local procedure OnBeforeRunIteration(var AITTestMethodLine: Record "AIT Test Method Line"; var AITTestSuite: Record "AIT Test Suite"; var RunAllTests: Boolean; var UpdateTestSuite: Boolean)
+    begin
+        RunAllTests := false;
+        UpdateTestSuite := false;
+    end;
+
+    [EventSubscriber(ObjectType::Codeunit, Codeunit::"AIT Test Run Iteration", 'OnBeforeRunTestMethodLine', '', false, false)]
+    local procedure OnBeforeRunTestMethodLine(var TestMethodLine: Record "Test Method Line")
+    begin
+        TestMethodLine.SetRange("Data Input Group Code", TestInputGroupCode);
+        TestMethodLine.SetRange("Data Input", TestInputCode);
+        TestMethodLine.SetRange("Line Type", TestMethodLine."Line Type"::Function);
+    end;
+
+}