diff --git a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al index 96b81d3e27..6f11b90cfd 100644 --- a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al +++ b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al @@ -25,6 +25,8 @@ permissionset 149031 "AI Test Toolkit - Obj" codeunit "AIT Run History" = X, xmlport "AIT Test Suite Import/Export" = X, page "AIT CommandLine Card" = X, + page "AIT Column Mappings" = X, + page "AIT Evaluators" = X, page "AIT Test Data" = X, page "AIT Test Data Compare" = X, page "AIT Batch Run Dialog" = X, diff --git a/src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al index dd68b22f82..4fc1b8f1be 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al @@ -94,6 +94,60 @@ codeunit 149044 "AIT Test Context" AITTestContextImpl.SetAnswerForQnAEvaluation(Answer); end; + /// + /// Sets the query and respone for a single-turn evaluation. + /// Optionally, a context can be provided. + /// + /// The query as text. + /// The response as text. + /// The context as text. + procedure SetQueryResponse(Query: Text; Response: Text; Context: Text) + begin + AITTestContextImpl.SetQueryResponse(Query, Response, Context); + end; + + /// + /// Sets the query and response for a single-turn evaluation. + /// + /// The query as text. + /// The response as text. + procedure SetQueryResponse(Query: Text; Response: Text) + begin + AITTestContextImpl.SetQueryResponse(Query, Response, ''); + end; + + /// + /// Adds a message to the current test iteration. + /// This is used for multi-turn tests to add messages to the output. + /// + /// The content of the message. + /// The role of the message (e.g., 'user', 'assistant'). + /// The context of the message. + procedure AddMessage(Content: Text; Role: Text; Context: Text) + begin + AITTestContextImpl.AddMessage(Content, Role, Context); + end; + + /// + /// Adds a message to the current test iteration. + /// This is used for multi-turn tests to add messages to the output. + /// + /// The content of the message. + /// The role of the message (e.g., 'user', 'assistant'). + procedure AddMessage(Content: Text; Role: Text) + begin + AITTestContextImpl.AddMessage(Content, Role, ''); + end; + + /// + /// Sets the test output for the current iteration. + /// + /// The test output. + procedure SetTestOutput(TestOutputJson: Codeunit "Test Output Json") + begin + AITTestContextImpl.SetTestOutput(TestOutputJson); + end; + /// /// Sets the test output for the current iteration. /// @@ -123,6 +177,35 @@ codeunit 149044 "AIT Test Context" AITTestContextImpl.SetTestMetric(TestMetric); end; + /// + /// Sets the accuracy of the test. + /// + /// The accuracy as a decimal between 0 and 1. + procedure SetAccuracy(Accuracy: Decimal) + begin + AITTestContextImpl.SetAccuracy(Accuracy); + end; + + /// + /// Gets the AITTestSuite associated with the run. + /// + /// AITTestSuite associated with the run. + procedure GetAITTestSuite(var AITTestSuite: Record "AIT Test Suite") + begin + AITTestContextImpl.GetAITTestSuite(AITTestSuite); + end; + + /// + /// Integration event that is raised after a test run is completed. + /// + /// The code of the test run. + /// The version of the test run. + /// The tag of the test run. + [IntegrationEvent(false, false)] + internal procedure OnAfterRunComplete(Code: Code[10]; Version: Integer; Tag: Text[20]) + begin + end; + var AITTestContextImpl: Codeunit "AIT Test Context Impl."; } \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al index 1499ffb8df..1abf5da3f7 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al @@ -18,9 +18,13 @@ codeunit 149043 "AIT Test Context Impl." var AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; GlobalTestOutputJson: Codeunit "Test Output Json"; + GlobalAccuracy: Decimal; CurrentTurn: Integer; NumberOfTurns: Integer; IsMultiTurn: Boolean; + AccuracySetManually: Boolean; + AccuracyErr: Label 'Accuracy must be between 0 and 1.'; + OnlySingleTurnErr: Label 'A query-and-response pair cannot be used in multi-turn tests. Use AddMessage instead.'; AnswerTok: Label 'answer', Locked = true; ContextTok: Label 'context', Locked = true; GroundTruthTok: Label 'ground_truth', Locked = true; @@ -29,6 +33,12 @@ codeunit 149043 "AIT Test Context Impl." TestSetupTok: Label 'test_setup', Locked = true; QuestionTok: Label 'question', Locked = true; TurnsTok: Label 'turns', Locked = true; + MessagesTok: Label 'messages', Locked = true; + QueryTok: Label 'query', Locked = true; + ResponseTok: Label 'response', Locked = true; + RoleTok: Label 'role', Locked = true; + ContentTok: Label 'content', Locked = true; + ConversationTok: Label 'conversation', Locked = true; /// /// Returns the Test Input value as Test Input Json Codeunit from the input dataset for the current iteration. @@ -103,6 +113,64 @@ codeunit 149043 "AIT Test Context Impl." SetSuiteTestOutput(CurrentTestOutputJson.ToText()); end; + /// + /// Sets the query and respone for a single-turn evaluation. + /// Optionally, a context can be provided. + /// + /// The query as text. + /// The response as text. + /// The context as text. + procedure SetQueryResponse(Query: Text; Response: Text; Context: Text) + var + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + CurrentTestOutputJson: Codeunit "Test Output Json"; + TestOutputCU: Codeunit "Test Output"; + begin + if IsMultiTurn then + Error(OnlySingleTurnErr); + + CurrentTestOutputJson.Initialize(); + CurrentTestOutputJson.Add(QueryTok, Query); + CurrentTestOutputJson.Add(ResponseTok, Response); + + if Context <> '' then + CurrentTestOutputJson.Add(ContextTok, Context); + + TestOutputCU.TestData().Initialize(CurrentTestOutputJson.ToText()); + + AITTestSuiteMgt.SetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestOutputCU.Testdata().ToText()); + end; + + /// + /// Adds a message to the current test iteration. + /// This is used for multi-turn tests to add messages to the output. + /// + /// The content of the message. + /// The role of the message (e.g., 'user', 'assistant'). + /// The context of the message (can be blank). + procedure AddMessage(Content: Text; Role: Text; Context: Text) + var + CurrentTestOutputJson: Codeunit "Test Output Json"; + begin + CurrentTestOutputJson.Initialize(); + CurrentTestOutputJson.Add(ContentTok, Content); + CurrentTestOutputJson.Add(RoleTok, Role); + + if Context <> '' then + CurrentTestOutputJson.Add(ContextTok, Context); + + AddMessageToOutput(CurrentTestOutputJson.ToText()); + end; + + /// + /// Sets the test output for the current iteration. + /// + /// The test output. + procedure SetTestOutput(TestOutputJson: Codeunit "Test Output Json") + begin + SetSuiteTestOutput(TestOutputJson.ToText()); + end; + /// /// Sets the test output for the current iteration. /// @@ -146,6 +214,34 @@ codeunit 149043 "AIT Test Context Impl." SetSuiteTestOutput(CurrentTestOutputJson.ToText()); end; + /// + /// Sets the accuracy of the test. + /// + /// The accuracy as a decimal between 0 and 1. + procedure SetAccuracy(Accuracy: Decimal) + begin + if (Accuracy < 0) or (Accuracy > 1) then + Error(AccuracyErr); + + AccuracySetManually := true; + GlobalAccuracy := Accuracy; + end; + + /// + /// Gets the accuracy of the test. Can only be retrieved if the accuracy of the test was already set manually. + /// + /// The accuracy as a decimal between 0 and 1. + /// True if it was possible to get the accuracy, false otherwise. + procedure GetAccuracy(var Accuracy: Decimal): Boolean + begin + if AccuracySetManually then begin + Accuracy := GlobalAccuracy; + exit(true); + end; + + exit(false); + end; + /// /// Sets to next turn for multiturn testing. /// @@ -164,7 +260,7 @@ codeunit 149043 "AIT Test Context Impl." end; /// - /// Gets the current turn for multiturn testing. Turns start from turn 0. + /// Gets the current turn for multiturn testing. Turns start from turn 1. /// /// The current turn number. procedure GetCurrentTurn(): Integer @@ -172,6 +268,26 @@ codeunit 149043 "AIT Test Context Impl." exit(CurrentTurn); end; + /// + /// Gets the total number of turns for multiturn testing. + /// + /// The total number of turns for the line. + procedure GetNumberOfTurns(): Integer + begin + exit(NumberOfTurns); + end; + + /// + /// Returns the AITTestSuite associated with the run. + /// + /// AITTestSuite associated with the run. + procedure GetAITTestSuite(var AITTestSuite: Record "AIT Test Suite") + var + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + begin + AITTestRunIteration.GetAITTestSuite(AITTestSuite); + end; + /// /// This method starts the scope of the Run Procedure scenario. /// @@ -205,12 +321,16 @@ codeunit 149043 "AIT Test Context Impl." TestInput: Codeunit "Test Input"; TurnsInputJson: Codeunit "Test Input Json"; begin - CurrentTurn := 0; + AccuracySetManually := false; + GlobalAccuracy := 0; + CurrentTurn := 1; GlobalTestOutputJson.Initialize(); TurnsInputJson := TestInput.GetTestInput().ElementExists(TurnsTok, IsMultiTurn); if IsMultiTurn then - NumberOfTurns := TurnsInputJson.GetElementCount() - 1; + NumberOfTurns := TurnsInputJson.GetElementCount() + else + NumberOfTurns := 1; end; /// @@ -223,11 +343,29 @@ codeunit 149043 "AIT Test Context Impl." TestInput: Codeunit "Test Input"; begin if IsMultiTurn then - TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn).Element(ElementName) + TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn - 1).Element(ElementName) else TestInputJson := TestInput.GetTestInput(ElementName); end; + /// + /// Adds a message to the test output for the current iteration. + /// + local procedure AddMessageToOutput(Output: Text) + var + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + TestOutputCU: Codeunit "Test Output"; + begin + if not TestOutputCU.TestData().ElementExists(ConversationTok) then begin + TestOutputCU.TestData().Add(ConversationTok, ''); + TestOutputCU.TestData().Element(ConversationTok).AddArray(MessagesTok); + end; + + TestOutputCU.TestData().Element(ConversationTok).Element(MessagesTok).Add(Output); + + AITTestSuiteMgt.SetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestOutputCU.Testdata().ToText()); + end; + /// /// Sets the test output for the current iteration. /// diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunInputHandler.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunInputHandler.Codeunit.al new file mode 100644 index 0000000000..900c15e6a9 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/AITTestRunInputHandler.Codeunit.al @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.TestTools.TestRunner; + +codeunit 149045 "AIT Test Run Input Handler" +{ + SingleInstance = true; + EventSubscriberInstance = Manual; + InherentEntitlements = X; + InherentPermissions = X; + + var + TestInputGroupCode: Code[100]; + TestInputCode: Code[100]; + + internal procedure SetInput(InputGroupCode: Code[100]; InputCode: Code[100]) + begin + TestInputGroupCode := InputGroupCode; + TestInputCode := InputCode; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"AIT Test Run Iteration", 'OnBeforeRunIteration', '', false, false)] + local procedure OnBeforeRunIteration(var AITTestMethodLine: Record "AIT Test Method Line"; var AITTestSuite: Record "AIT Test Suite"; var RunAllTests: Boolean; var UpdateTestSuite: Boolean) + begin + RunAllTests := false; + UpdateTestSuite := false; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"AIT Test Run Iteration", 'OnBeforeRunTestMethodLine', '', false, false)] + local procedure OnBeforeRunTestMethodLine(var TestMethodLine: Record "Test Method Line") + begin + TestMethodLine.SetRange("Data Input Group Code", TestInputGroupCode); + TestMethodLine.SetRange("Data Input", TestInputCode); + TestMethodLine.SetRange("Line Type", TestMethodLine."Line Type"::Function); + end; + +} diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al index 4565bd168a..bf99de97ec 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al @@ -20,7 +20,12 @@ codeunit 149042 "AIT Test Run Iteration" ActiveAITTestSuite: Record "AIT Test Suite"; GlobalTestMethodLine: Record "Test Method Line"; NoOfInsertedLogEntries: Integer; + UpdateTestSuite: Boolean; + RunAllTests: Boolean; GlobalAITokenUsedByLastTestMethodLine: Integer; + GlobalNumberOfTurnsForLastTestMethodLine: Integer; + GlobalNumberOfTurnsPassedForLastTestMethodLine: Integer; + GlobalTestAccuracy: Decimal; GlobalSessionAITokenUsed: Integer; trigger OnRun() @@ -31,6 +36,8 @@ codeunit 149042 "AIT Test Run Iteration" NoOfInsertedLogEntries := 0; GlobalAITokenUsedByLastTestMethodLine := 0; + UpdateTestSuite := true; + RunAllTests := true; InitializeAITTestMethodLineForRun(Rec, ActiveAITTestSuite); SetAITTestSuite(ActiveAITTestSuite); @@ -52,7 +59,7 @@ codeunit 149042 "AIT Test Run Iteration" var AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; begin - OnBeforeRunIteration(AITTestSuite, AITTestMethodLine); + OnBeforeRunIteration(AITTestSuite, AITTestMethodLine, RunAllTests, UpdateTestSuite); RunIteration(AITTestMethodLine); Commit(); @@ -66,14 +73,23 @@ codeunit 149042 "AIT Test Run Iteration" TestSuiteMgt: Codeunit "Test Suite Mgt."; begin AITTestMethodLine.Find(); - AITALTestSuiteMgt.UpdateALTestSuite(AITTestMethodLine); + + if UpdateTestSuite then + AITALTestSuiteMgt.UpdateALTestSuite(AITTestMethodLine); + SetAITTestMethodLine(AITTestMethodLine); TestMethodLine.SetRange("Test Codeunit", AITTestMethodLine."Codeunit ID"); TestMethodLine.SetRange("Test Suite", AITTestMethodLine."AL Test Suite"); TestMethodLine.SetRange("Line Type", TestMethodLine."Line Type"::Codeunit); + OnBeforeRunTestMethodLine(TestMethodLine); + TestMethodLine.FindFirst(); - TestSuiteMgt.RunAllTests(TestMethodLine); + + if RunAllTests then + TestSuiteMgt.RunAllTests(TestMethodLine) + else + TestSuiteMgt.RunSelectedTests(TestMethodLine); end; procedure GetAITTestSuiteTag(): Text[20] @@ -124,8 +140,28 @@ codeunit 149042 "AIT Test Run Iteration" exit(GlobalAITokenUsedByLastTestMethodLine); end; + procedure GetNumberOfTurnsForLastTestMethodLine(): Integer + begin + exit(GlobalNumberOfTurnsForLastTestMethodLine); + end; + + procedure GetNumberOfTurnsPassedForLastTestMethodLine(): Integer + begin + exit(GlobalNumberOfTurnsPassedForLastTestMethodLine); + end; + + procedure GetAccuracyForLastTestMethodLine(): Decimal + begin + exit(GlobalTestAccuracy); + end; + + [InternalEvent(false)] + procedure OnBeforeRunIteration(var AITTestSuite: Record "AIT Test Suite"; var AITTestMethodLine: Record "AIT Test Method Line"; var RunAllTests: Boolean; var UpdateTestSuite: Boolean) + begin + end; + [InternalEvent(false)] - procedure OnBeforeRunIteration(var AITTestSuite: Record "AIT Test Suite"; var AITTestMethodLine: Record "AIT Test Method Line") + procedure OnBeforeRunTestMethodLine(var TestMethodLine: Record "Test Method Line") begin end; @@ -144,6 +180,14 @@ codeunit 149042 "AIT Test Run Iteration" // Update AI Token Consumption GlobalAITokenUsedByLastTestMethodLine := 0; + + // Update Turns + GlobalNumberOfTurnsPassedForLastTestMethodLine := 0; + GlobalNumberOfTurnsForLastTestMethodLine := 1; + + // Update Test Accuracy + GlobalTestAccuracy := 0; + GlobalSessionAITokenUsed := AOAIToken.GetTotalServerSessionTokensConsumed(); AITContextCU.StartRunProcedureScenario(); @@ -154,6 +198,7 @@ codeunit 149042 "AIT Test Run Iteration" var AITContextCU: Codeunit "AIT Test Context Impl."; AOAIToken: Codeunit "AOAI Token"; + Accuracy: Decimal; begin if ActiveAITTestSuite.Code = '' then exit; @@ -166,6 +211,19 @@ codeunit 149042 "AIT Test Run Iteration" // Update AI Token Consumption GlobalAITokenUsedByLastTestMethodLine := AOAIToken.GetTotalServerSessionTokensConsumed() - GlobalSessionAITokenUsed; + // Update Turns + GlobalNumberOfTurnsForLastTestMethodLine := AITContextCU.GetNumberOfTurns(); + GlobalNumberOfTurnsPassedForLastTestMethodLine := AITContextCU.GetCurrentTurn(); + + if not IsSuccess then + GlobalNumberOfTurnsPassedForLastTestMethodLine -= 1; + + // Update Test Accuracy + if AITContextCU.GetAccuracy(Accuracy) then + GlobalTestAccuracy := Accuracy + else + GlobalTestAccuracy := GlobalNumberOfTurnsPassedForLastTestMethodLine / GlobalNumberOfTurnsForLastTestMethodLine; + AITContextCU.EndRunProcedureScenario(CurrentTestMethodLine, IsSuccess); Commit(); end; diff --git a/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMapping.Table.al b/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMapping.Table.al new file mode 100644 index 0000000000..b08ac8de72 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMapping.Table.al @@ -0,0 +1,58 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ +namespace System.TestTools.AITestToolkit; + +table 149038 "AIT Column Mapping" +{ + DataClassification = SystemMetadata; + InherentEntitlements = RIMDX; + InherentPermissions = RIMDX; + ReplicateData = false; + DrillDownPageId = "AIT Column Mappings"; + + fields + { + field(1; "Test Suite Code"; Code[10]) + { + Caption = 'Test Suite Code'; + ToolTip = 'Specifies the code of the test suite.'; + DataClassification = SystemMetadata; + TableRelation = "AIT Test Suite".Code; + ValidateTableRelation = true; + } + + field(2; "Test Method Line"; Integer) + { + Caption = 'Test Method Line'; + ToolTip = 'Specifies the line number of the test method.'; + DataClassification = SystemMetadata; + TableRelation = "AIT Test Method Line"."Line No."; + ValidateTableRelation = true; + } + + field(10; "Column"; Text[1024]) + { + DataClassification = SystemMetadata; + Caption = 'Column'; + ToolTip = 'Specifies the column that needs to be mapped from the output.'; + } + + field(11; "Target Column"; Text[1024]) + { + DataClassification = SystemMetadata; + Caption = 'Target Column'; + ToolTip = 'Specifies the target column that needs to be mapped to externally.'; + } + + } + + keys + { + key(PK; "Test Suite Code", "Test Method Line", Column, "Target Column") + { + Clustered = true; + } + } +} \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMappings.Page.al b/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMappings.Page.al new file mode 100644 index 0000000000..ead590ba39 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/Evaluation/AITColumnMappings.Page.al @@ -0,0 +1,48 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ +namespace System.TestTools.AITestToolkit; + +page 149044 "AIT Column Mappings" +{ + PageType = List; + ApplicationArea = All; + SourceTable = "AIT Column Mapping"; + + layout + { + area(Content) + { + repeater(ColumnMappings) + { + field("Test Suite Code"; Rec."Test Suite Code") + { + Editable = false; + ApplicationArea = All; + Caption = 'Test Suite Code'; + ToolTip = 'Specifies the code of the test suite.'; + } + field("Test Method Line"; Rec."Test Method Line") + { + Editable = false; + ApplicationArea = All; + Caption = 'Test Method Line'; + ToolTip = 'Specifies the line number of the test method.'; + } + field(Column; Rec.Column) + { + ApplicationArea = All; + Caption = 'Column'; + ToolTip = 'Specifies the column from the test output data to use in evaluation.'; + } + field("Target Column"; Rec."Target Column") + { + ApplicationArea = All; + Caption = 'Target Column'; + ToolTip = 'Specifies the target column from the test output data to use in evaluation.'; + } + } + } + } +} \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluator.Table.al b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluator.Table.al new file mode 100644 index 0000000000..bf5be212e5 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluator.Table.al @@ -0,0 +1,59 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ +namespace System.TestTools.AITestToolkit; + +table 149039 "AIT Evaluator" +{ + DataClassification = SystemMetadata; + InherentEntitlements = RIMDX; + InherentPermissions = RIMDX; + ReplicateData = false; + DrillDownPageId = "AIT Evaluators"; + + fields + { + field(1; "Test Suite Code"; Code[10]) + { + Caption = 'Test Suite Code'; + ToolTip = 'Specifies the code of the test suite.'; + DataClassification = SystemMetadata; + TableRelation = "AIT Test Suite".Code; + ValidateTableRelation = true; + } + + field(2; "Test Method Line"; Integer) + { + Caption = 'Test Method Line'; + ToolTip = 'Specifies the line number of the test method.'; + DataClassification = SystemMetadata; + TableRelation = "AIT Test Method Line"."Line No."; + ValidateTableRelation = true; + } + + field(3; "Evaluator Type"; Enum "AIT Evaluator Type") + { + DataClassification = SystemMetadata; + Caption = 'Evaluator Type'; + ToolTip = 'Specifies the type of evaluator.'; + } + + field(10; Evaluator; Text[100]) + { + DataClassification = SystemMetadata; + Caption = 'Evaluator'; + ToolTip = 'Specifies the evaluator to use in the test suite.'; + } + + + } + + keys + { + key(PK; "Test Suite Code", "Test Method Line", Evaluator) + { + Clustered = true; + } + } +} \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluatorType.Enum.al b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluatorType.Enum.al new file mode 100644 index 0000000000..c467834fa8 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluatorType.Enum.al @@ -0,0 +1,18 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ +namespace System.TestTools.AITestToolkit; + +enum 149039 "AIT Evaluator Type" +{ + Extensible = true; + + value(0; "Built-in") + { + } + + value(1; Custom) + { + } +} \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluators.Page.al b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluators.Page.al new file mode 100644 index 0000000000..0969dbbaa4 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/Evaluation/AITEvaluators.Page.al @@ -0,0 +1,64 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ +namespace System.TestTools.AITestToolkit; + +page 149045 "AIT Evaluators" +{ + PageType = List; + ApplicationArea = All; + SourceTable = "AIT Evaluator"; + + layout + { + area(Content) + { + repeater(Evaluators) + { + field("Test Suite Code"; Rec."Test Suite Code") + { + Editable = false; + ApplicationArea = All; + Caption = 'Test Suite Code'; + ToolTip = 'Specifies the code of the test suite.'; + Visible = false; + } + field("Test Method Line"; Rec."Test Method Line") + { + Editable = false; + ApplicationArea = All; + Caption = 'Test Method Line'; + ToolTip = 'Specifies the line number of the test method.'; + Visible = false; + } + field(Evaluator; Rec.Evaluator) + { + ApplicationArea = All; + Caption = 'Evaluator'; + ToolTip = 'Specifies the evaluator to use in the test suite.'; + } + field(EvaluatorType; Rec."Evaluator Type") + { + ApplicationArea = All; + Caption = 'Evaluator Type'; + ToolTip = 'Specifies the type of evaluator.'; + } + } + } + + } + + trigger OnNewRecord(BelowxRec: Boolean) + begin + Rec."Test Method Line" := TestMethodLineNo; + end; + + var + TestMethodLineNo: Integer; + + internal procedure SetTestMethodLine(LineNo: Integer) + begin + TestMethodLineNo := LineNo; + end; +} \ No newline at end of file diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al index 123d63634e..1f05d0929a 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al @@ -12,7 +12,7 @@ page 149033 "AIT Log Entries" ApplicationArea = All; Editable = false; SourceTable = "AIT Log Entry"; - Extensible = false; + Extensible = true; UsageCategory = None; layout @@ -68,6 +68,24 @@ page 149033 "AIT Log Entries" { StyleExpr = StatusStyleExpr; } + field(Accuracy; Rec."Test Method Line Accuracy") + { + AutoFormatType = 0; + } + field("No. of Turns Passed"; Rec."No. of Turns Passed") + { + Visible = false; + } + field("No. of Turns"; Rec."No. of Turns") + { + Visible = false; + } + field(TurnsText; TurnsText) + { + StyleExpr = TurnsStyleExpr; + Caption = 'No. of Turns Passed'; + ToolTip = 'Specifies the number of turns that passed out of the total number of turns.'; + } field("Orig. Status"; Rec."Original Status") { Visible = false; @@ -253,11 +271,34 @@ page 149033 "AIT Log Entries" Page.Run(Page::"AIT Test Data Compare", Rec); end; } + + action(RerunTest) + { + Caption = 'Rerun test'; + Image = Redo; + ToolTip = 'Rerun the test for the selected line.'; + + trigger OnAction() + var + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; + Version: Integer; + begin + Version := AITTestSuiteMgt.RerunTest(Rec); + + Rec.Reset(); + Rec.SetRange(Version, Version); + CurrPage.Update(false); + end; + } } area(Promoted) { group(Category_Process) { + + actionref("RerunTest_Promoted"; "RerunTest") + { + } actionref(DeleteAll_Promoted; DeleteAll) { } @@ -287,20 +328,26 @@ page 149033 "AIT Log Entries" ClickToShowLbl: Label 'Show data input'; DoYouWantToDeleteQst: Label 'Do you want to delete all entries within the filter?'; InputText: Text; + TurnsText: Text; OutputText: Text; ErrorMessage: Text; ErrorCallStack: Text; StatusStyleExpr: Text; + TurnsStyleExpr: Text; TestRunDuration: Duration; IsFilteredToErrors: Boolean; ShowSensitiveData: Boolean; trigger OnAfterGetRecord() + var + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; begin TestRunDuration := Rec."Duration (ms)"; + TurnsText := AITTestSuiteMgt.GetTurnsAsText(Rec); SetInputOutputDataFields(); SetErrorFields(); SetStatusStyleExpr(); + SetTurnsStyleExpr(); end; local procedure SetStatusStyleExpr() @@ -315,6 +362,18 @@ page 149033 "AIT Log Entries" end; end; + local procedure SetTurnsStyleExpr() + begin + case Rec."No. of Turns Passed" of + Rec."No. of Turns": + TurnsStyleExpr := 'Favorable'; + 0: + TurnsStyleExpr := 'Unfavorable'; + else + TurnsStyleExpr := 'Ambiguous'; + end; + end; + local procedure SetErrorFields() begin ErrorMessage := ''; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al index efe7955c0e..0508f4b4bf 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al @@ -15,8 +15,8 @@ table 149034 "AIT Log Entry" DrillDownPageId = "AIT Log Entries"; LookupPageId = "AIT Log Entries"; DataCaptionFields = "Codeunit Name", "Procedure Name", "Test Input Code"; - Extensible = false; - Access = Internal; + Extensible = true; + Access = Public; ReplicateData = false; fields @@ -159,6 +159,23 @@ table 149034 "AIT Log Entry" { Caption = 'Output Data'; } + field(40; "No. of Turns"; Integer) + { + Caption = 'Total number of turns'; + ToolTip = 'Specifies the total number of turns.'; + } + field(41; "No. of Turns Passed"; Integer) + { + Caption = 'Number of turns passed'; + ToolTip = 'Specifies the number of turns passed.'; + } + field(45; "Test Method Line Accuracy"; Decimal) + { + Caption = 'Test Method Line Accuracy'; + ToolTip = 'Specifies the accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually in the test.'; + AutoFormatType = 0; + + } field(50; "Tokens Consumed"; Integer) { Caption = 'Total Tokens Consumed'; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al index 456639fbb0..071e3c5b15 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al @@ -121,6 +121,12 @@ page 149032 "AIT Run History" AITLogEntryCodeunit.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No. Filter", Rec.Version); end; } + field("Accuracy - By Version"; Rec."Accuracy Per Version") + { + Visible = ViewBy = ViewBy::Version; + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the version.'; + } field("Duration - By Version"; Rec."Total Duration (ms)") { Visible = ViewBy = ViewBy::Version; @@ -161,6 +167,12 @@ page 149032 "AIT Run History" AITLogEntryCodeunit.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No. Filter", Rec.Tag); end; } + field("Accuracy - By Tag"; Rec."Accuracy - By Tag") + { + Visible = ViewBy = ViewBy::Tag; + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the tag.'; + } field("Duration - By Tag"; Rec."Total Duration (ms) - By Tag") { Visible = ViewBy = ViewBy::Tag; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al b/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al index 8194182f71..cf86c3f543 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al @@ -73,6 +73,15 @@ table 149036 "AIT Run History" FieldClass = FlowField; CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), Version = field("Version"), "Test Method Line No." = field("Line No. Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); } + field(14; "Accuracy Per Version"; Decimal) + { + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the version.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + AutoFormatType = 0; + } field(20; "No. of Tests Executed - By Tag"; Integer) { Caption = 'No. of Tests Executed'; @@ -105,6 +114,15 @@ table 149036 "AIT Run History" FieldClass = FlowField; CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); } + field(24; "Accuracy - By Tag"; Decimal) + { + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the tag.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + AutoFormatType = 0; + } } keys diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Codeunit.al b/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Codeunit.al index 35506c2efd..96843dd984 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Codeunit.al @@ -11,6 +11,9 @@ codeunit 149035 "AIT Test Data" { Access = Internal; + var + TurnsLbl: Label '%1: %2
', Comment = '%1 = The turn number as an integer, %2 = The data from that turn', Locked = true; + procedure UpdateTestInput(TestInput: Text; TestInputView: Enum "AIT Test Input - View"): Text var TestData: Codeunit "Test Input Json"; @@ -21,15 +24,15 @@ codeunit 149035 "AIT Test Data" TestInputView::"Full Input": exit(TestInput); TestInputView::Question: - exit(GetTestDataElement('question', TestData)); + exit(FilterToElement('question', TestData)); TestInputView::Context: - exit(GetTestDataElement('context', TestData)); + exit(FilterToElement('context', TestData)); TestInputView::"Test Setup": - exit(GetTestDataElement('test_setup', TestData)); + exit(FilterToElement('test_setup', TestData)); TestInputView::"Ground Truth": - exit(GetTestDataElement('ground_truth', TestData)); + exit(FilterToElement('ground_truth', TestData)); TestInputView::"Expected Data": - exit(GetTestDataElement('expected_data', TestData)); + exit(FilterToElement('expected_data', TestData)); else exit(''); end; @@ -45,13 +48,13 @@ codeunit 149035 "AIT Test Data" TestOutputView::"Full Output": exit(TestOutput); TestOutputView::Answer: - exit(GetTestDataElement('answer', TestData)); + exit(FilterToElement('answer', TestData)); TestOutputView::Question: - exit(GetTestDataElement('question', TestData)); + exit(FilterToElement('question', TestData)); TestOutputView::Context: - exit(GetTestDataElement('context', TestData)); + exit(FilterToElement('context', TestData)); TestOutputView::"Ground Truth": - exit(GetTestDataElement('ground_truth', TestData)); + exit(FilterToElement('ground_truth', TestData)); else exit(''); end; @@ -65,20 +68,38 @@ codeunit 149035 "AIT Test Data" TestData.Initialize(TestDataText); end; - local procedure GetTestDataElement(ElementName: Text; TestData: Codeunit "Test Input Json"): Text + local procedure FilterToElement(ElementName: Text; TestData: Codeunit "Test Input Json"): Text var - ElementTestDataJson: Codeunit "Test Input Json"; - ElementExists: Boolean; + TurnsDataJson: Codeunit "Test Input Json"; + ElementJson: Codeunit "Test Input Json"; + TextBuilder: TextBuilder; + IsMultiTurn: Boolean; + NumberOfTurns: Integer; + I: Integer; begin - ElementTestDataJson := TestData.ElementExists('turns', ElementExists); + TurnsDataJson := TestData.ElementExists('turns', IsMultiTurn); + + if not IsMultiTurn then + exit(GetTestDataElement(ElementName, TestData)); + + NumberOfTurns := TurnsDataJson.GetElementCount(); + for I := 0 to NumberOfTurns - 1 do begin + ElementJson := TurnsDataJson.ElementAt(I); + TextBuilder.AppendLine(StrSubstNo(TurnsLbl, I, GetTestDataElement(ElementName, ElementJson))); + end; - if ElementExists then - TestData := ElementTestDataJson; + exit(TextBuilder.ToText()); + end; - ElementTestDataJson := TestData.ElementExists(ElementName, ElementExists); + local procedure GetTestDataElement(ElementName: Text; var TestData: Codeunit "Test Input Json"): Text + var + ElementJson: Codeunit "Test Input Json"; + ElementExists: Boolean; + begin + ElementJson := TestData.ElementExists(ElementName, ElementExists); - if ElementExists and (ElementTestDataJson.ToText() <> '{}') then - exit(ElementTestDataJson.ToText()) + if ElementExists and (ElementJson.ToText() <> '{}') then + exit(ElementJson.ToText()) else exit(''); end; diff --git a/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Page.al b/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Page.al index f0084d0358..7742a272b1 100644 --- a/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Page.al +++ b/src/Tools/AI Test Toolkit/src/Logs/AITTestData.Page.al @@ -34,7 +34,7 @@ page 149041 "AIT Test Data" } - internal procedure SetTestData(Text: Text) + procedure SetTestData(Text: Text) begin TestDataText := Text; CurrPage.Update(false); diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al index d533d5c52a..418130c4df 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al @@ -194,6 +194,26 @@ codeunit 149037 "AIT AL Test Suite Mgt" end; end; + procedure GetTestOutputAsJson(var AITLogEntry: Record "AIT Log Entry"): JsonArray + var + OutputJsonArray: JsonArray; + OutputJson: JsonObject; + TestOutput: Text; + begin + AITLogEntry.SetLoadFields("Test Suite Code", "Output Data"); + AITLogEntry.ReadIsolation := IsolationLevel::ReadUncommitted; + if AITLogEntry.FindSet() then + repeat + TestOutput := AITLogEntry.GetOutputBlob(); + if TestOutput <> '' then begin + OutputJson.ReadFrom(TestOutput); + OutputJsonArray.Add(OutputJson); + end; + until AITLogEntry.Next() = 0; + + exit(OutputJsonArray); + end; + /// /// Import the Test Input Dataset from an InStream of a dataset in a supported format. /// Overwrite the dataset if the dataset with same filename is already imported by the same app diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al index 7f582e6ed4..12982d679c 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al @@ -12,8 +12,8 @@ table 149032 "AIT Test Method Line" { Caption = 'AI Test Method Line'; DataClassification = SystemMetadata; - Extensible = false; - Access = Internal; + Extensible = true; + Access = Public; ReplicateData = false; fields @@ -166,6 +166,31 @@ table 149032 "AIT Test Method Line" FieldClass = FlowField; CalcFormula = count("AIT Log Entry" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Base Version Filter"))); } + field(40; "No. of Turns"; Integer) + { + Caption = 'No. of Turns Executed'; + ToolTip = 'Specifies the total number of turns for the test line.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."No. of Turns" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } + field(41; "No. of Turns Passed"; Integer) + { + Caption = 'No. of Turns Passed'; + ToolTip = 'Specifies the total number of passed turns for the test line.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = sum("AIT Log Entry"."No. of Turns Passed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + } + field(45; "Test Method Line Accuracy"; Decimal) + { + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually by the test.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No."), Version = field("Version Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + AutoFormatType = 0; + } field(101; "AL Test Suite"; Code[10]) { Caption = 'AL Test Suite'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al index 7af836cbed..0c90ff2422 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al @@ -13,7 +13,7 @@ page 149034 "AIT Test Method Lines" SourceTable = "AIT Test Method Line"; AutoSplitKey = true; DelayedInsert = true; - Extensible = false; + Extensible = true; UsageCategory = None; layout @@ -46,6 +46,25 @@ page 149034 "AIT Test Method Lines" field(Description; Rec.Description) { } + field("Evaluation Setup"; EvaluationSetupTxt) + { + ApplicationArea = All; + Caption = 'Evaluators'; + ToolTip = 'Specifies whether the evaluation is setup.'; + Editable = false; + + trigger OnAssistEdit() + var + AITEvaluator: Record "AIT Evaluator"; + AITEvaluatorPage: Page "AIT Evaluators"; + begin + AITEvaluator.SetRange("Test Suite Code", Rec."Test Suite Code"); + AITEvaluator.SetRange("Test Method Line", Rec."Line No."); + AITEvaluatorPage.SetTableView(AITEvaluator); + AITEvaluatorPage.SetTestMethodLine(Rec."Line No."); + AITEvaluatorPage.Run(); + end; + } field(Status; Rec.Status) { } @@ -73,6 +92,52 @@ page 149034 "AIT Test Method Lines" AITLogEntry.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No.", AITTestSuite.Version); end; } + field(Accuracy; Rec."Test Method Line Accuracy") + { + } + field(TurnsText; TurnsText) + { + Visible = false; + Editable = false; + Caption = 'No. of Turns Passed'; + ToolTip = 'Specifies the number of turns that passed out of the total number of turns.'; + + trigger OnDrillDown() + var + AITTestSuite: Record "AIT Test Suite"; + AITLogEntry: Codeunit "AIT Log Entry"; + begin + AITTestSuite.SetLoadFields("Base Version"); + AITTestSuite.Get(Rec."Test Suite Code"); + AITLogEntry.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No.", AITTestSuite."Base Version"); + end; + } + field("No. of Turns"; Rec."No. of Turns") + { + Visible = false; + } + field("No. of Turns Passed"; Rec."No. of Turns Passed") + { + Visible = false; + } + field("No. of Turns Failed"; Rec."No. of Turns" - Rec."No. of Turns Passed") + { + Visible = false; + Editable = false; + Caption = 'No. of Turns Failed'; + ToolTip = 'Specifies the number of failed turns of the test line.'; + Style = Unfavorable; + + trigger OnDrillDown() + var + AITTestSuite: Record "AIT Test Suite"; + AITLogEntry: Codeunit "AIT Log Entry"; + begin + AITTestSuite.SetLoadFields("Base Version"); + AITTestSuite.Get(Rec."Test Suite Code"); + AITLogEntry.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No.", AITTestSuite."Base Version"); + end; + } field("No. of Operations"; Rec."No. of Operations") { Visible = false; @@ -137,6 +202,7 @@ page 149034 "AIT Test Method Lines" Caption = 'Change in Duration (%)'; ToolTip = 'Specifies difference in average test execution time compared to the base version.'; Visible = false; + AutoFormatType = 0; } } } @@ -196,6 +262,8 @@ page 149034 "AIT Test Method Lines" AITTestSuite: Record "AIT Test Suite"; AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; NoLineSelectedErr: Label 'Select a line to compare'; + TurnsText: Text; + EvaluationSetupTxt: Text; trigger OnInsertRecord(BelowxRec: Boolean): Boolean begin @@ -205,6 +273,12 @@ page 149034 "AIT Test Method Lines" if AITTestSuite.Get(Rec."Test Suite Code") then; end; + trigger OnAfterGetRecord() + begin + EvaluationSetupTxt := AITTestSuiteMgt.GetEvaluationSetupText(CopyStr(Rec."Test Suite Code", 1, 10), Rec."Line No."); + TurnsText := AITTestSuiteMgt.GetTurnsAsText(Rec); + end; + local procedure GetAvg(NumIterations: Integer; TotalNo: Integer): Integer begin if NumIterations = 0 then diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al index 5096fadbbe..547b0fab5a 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Page.al @@ -14,7 +14,7 @@ page 149031 "AIT Test Suite" ApplicationArea = All; PageType = Document; SourceTable = "AIT Test Suite"; - Extensible = false; + Extensible = true; DataCaptionExpression = PageCaptionLbl + ' - ' + Rec."Code"; UsageCategory = None; @@ -77,6 +77,41 @@ page 149031 "AIT Test Suite" CurrPage.Update(true); end; } + group(Evaluation) + { + field("Evaluation Setup"; EvaluationSetupTxt) + { + ApplicationArea = All; + Caption = 'Evaluators'; + ToolTip = 'Specifies whether the evaluation is setup.'; + Editable = false; + + trigger OnAssistEdit() + var + AITEvaluator: Record "AIT Evaluator"; + AITEvaluatorPage: Page "AIT Evaluators"; + begin + AITEvaluator.SetRange("Test Suite Code", Rec.Code); + AITEvaluator.SetRange("Test Method Line", 0); + AITEvaluatorPage.SetTableView(AITEvaluator); + AITEvaluatorPage.SetTestMethodLine(0); + AITEvaluatorPage.Run(); + end; + } + field(Evaluators; Rec."Number of Evaluators") + { + ApplicationArea = All; + Caption = 'Number of Evaluators'; + ToolTip = 'Specifies evaluators for the evaluation.'; + } + + field("Column Mappings"; Rec."Number of Column Mappings") + { + ApplicationArea = All; + Caption = 'Column Mappings'; + ToolTip = 'Specifies column mappings for the evaluation.'; + } + } group(StatusGroup) { Caption = 'Suite Status'; @@ -127,6 +162,9 @@ page 149031 "AIT Test Suite" AITLogEntry.DrillDownFailedAITLogEntries(Rec.Code, 0, Rec.Version); end; } + field(Accuracy; Rec.Accuracy) + { + } field("No. of Operations"; Rec."No. of Operations") { Visible = false; @@ -313,6 +351,7 @@ page 149031 "AIT Test Suite" PageCaptionLbl: Label 'AI Test'; TestRunnerDisplayName: Text; InputDatasetChangedQst: Label 'You have modified the input dataset.\\Do you want to update the lines?'; + EvaluationSetupTxt: Text; trigger OnOpenPage() var @@ -333,6 +372,7 @@ page 149031 "AIT Test Suite" UpdateTotalDuration(); UpdateAverages(); TestRunnerDisplayName := TestSuiteMgt.GetTestRunnerDisplayName(Rec."Test Runner Id"); + EvaluationSetupTxt := AITTestSuiteMgt.GetEvaluationSetupText(Rec.Code, 0); end; local procedure UpdateTotalDuration() diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al index cf97553ad6..a53fde47df 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuite.Table.al @@ -13,9 +13,9 @@ table 149030 "AIT Test Suite" { Caption = 'AI Test Suite'; DataClassification = SystemMetadata; - Extensible = false; ReplicateData = false; - Access = Internal; + Extensible = true; + Access = Public; fields { @@ -161,6 +161,31 @@ table 149030 "AIT Test Suite" FieldClass = FlowField; CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Code"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); } + field(25; Accuracy; Decimal) + { + Caption = 'Accuracy'; + ToolTip = 'Specifies the average accuracy of the test suite. The accuracy is calculated as the percentage of turns that passed or can be set manually by the test.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Code"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> ''))); + AutoFormatType = 0; + } + field(30; "Number of Evaluators"; Integer) + { + Caption = 'Evaluators'; + ToolTip = 'Specifies the number of evaluators to use in the test suite.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = count("AIT Evaluator" where("Test Suite Code" = field("Code"))); + } + field(31; "Number of Column Mappings"; Integer) + { + Caption = 'Column Mappings'; + ToolTip = 'Specifies the number of evaluators to use in the test suite.'; + Editable = false; + FieldClass = FlowField; + CalcFormula = count("AIT Column Mapping" where("Test Suite Code" = field("Code"))); + } field(50; "Test Runner Id"; Integer) { Caption = 'Test Runner Id'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al index d7a56e48a2..3c16c35f61 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al @@ -67,6 +67,29 @@ xmlport 149031 "AIT Test Suite Import/Export" { Occurrence = Optional; } + tableelement(AITEvaluator; "AIT Evaluator") + { + LinkFields = "Test Suite Code" = field("Code"); + LinkTable = "AITSuite"; + MinOccurs = Zero; + XmlName = 'Evaluator'; + SourceTableView = where("Test Method Line" = const(0)); + + fieldattribute(Evaluator; AITEvaluator.Evaluator) + { + Occurrence = Required; + } + fieldattribute(Type; AITEvaluator."Evaluator Type") + { + Occurrence = Required; + } + + trigger OnAfterInitRecord() + begin + if SkipTestSuites.Contains(AITSuite.Code) then + currXMLport.Skip(); + end; + } tableelement(AITestMethodLine; "AIT Test Method Line") { LinkFields = "Test Suite Code" = field("Code"); @@ -86,10 +109,32 @@ xmlport 149031 "AIT Test Suite Import/Export" { Occurrence = Optional; } - textattribute(EvaluatorText) + tableelement(AITLineEvaluator; "AIT Evaluator") { - Occurrence = Optional; + LinkFields = "Test Suite Code" = field("Test Suite Code"), "Test Method Line" = field("Line No."); + LinkTable = AITestMethodLine; + MinOccurs = Zero; XmlName = 'Evaluator'; + + fieldattribute(Evaluator; AITLineEvaluator.Evaluator) + { + Occurrence = Required; + } + fieldattribute(Type; AITLineEvaluator."Evaluator Type") + { + Occurrence = Required; + } + + trigger OnAfterInitRecord() + begin + if SkipTestSuites.Contains(AITSuite.Code) then + currXMLport.Skip(); + end; + + trigger OnBeforeInsertRecord() + begin + AITLineEvaluator."Test Method Line" := AITestMethodLine."Line No."; + end; } trigger OnAfterInitRecord() diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index d25d95fdb9..752abb1342 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -28,7 +28,12 @@ codeunit 149034 "AIT Test Suite Mgt." CannotRunMultipleSuitesInParallelErr: Label 'There is already a test run in progress. You need to wait for it to finish or cancel it before starting a new test run.'; FeatureNameLbl: Label 'AI Test Toolkit', Locked = true; LineNoFilterLbl: Label 'Codeunit %1 "%2" (Input: %3)', Locked = true; + TurnsLbl: Label '%1/%2', Comment = '%1 - No. of turns that passed, %2 - Total no. of turns'; ConfirmCancelQst: Label 'This action will mark the run as Cancelled. Are you sure you want to continue?'; + TestMethodLineNotFoundErr: Label 'The test suite %1 does not contain the test line %2. Run the suite again.', Comment = '%1 = test suite code, %2 = line number'; + TestSuiteChangedErr: Label 'The test suite %1 has been changed since test line %2 was run. Run the suite again.', Comment = '%1 = test suite code, %2 = line number'; + NoEvaluatorsLbl: Label 'Configure...'; + procedure StartAITSuite(Iterations: Integer; var AITTestSuite: Record "AIT Test Suite") var @@ -55,7 +60,6 @@ codeunit 149034 "AIT Test Suite Mgt." local procedure RunAITests(AITTestSuite: Record "AIT Test Suite") var AITTestMethodLine: Record "AIT Test Method Line"; - AITRunHistory: Record "AIT Run History"; AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; FeatureTelemetry: Codeunit "Feature Telemetry"; FeatureTelemetryCD: Dictionary of [Text, Text]; @@ -89,10 +93,29 @@ codeunit 149034 "AIT Test Suite Mgt." RunAITestLine(AITTestMethodLine, true); until AITTestMethodLine.Next() = 0; - AITRunHistory."Test Suite Code" := AITTestSuite.Code; - AITRunHistory.Version := AITTestSuite.Version; - AITRunHistory.Tag := AITTestSuite.Tag; - AITRunHistory.Insert(); + LogRunHistory(AITTestSuite.Code, AITTestSuite.Version, AITTestSuite.Tag); + end; + + internal procedure RerunTest(var AITLogEntry: Record "AIT Log Entry"): Integer + var + AITTestSuite: Record "AIT Test Suite"; + AITTestMethodLineForLogEntry: Record "AIT Test Method Line"; + AITTestRunInputHandler: Codeunit "AIT Test Run Input Handler"; + begin + if not AITTestMethodLineForLogEntry.Get(AITLogEntry."Test Suite Code", AITLogEntry."Test Method Line No.") then + Error(TestMethodLineNotFoundErr, AITLogEntry."Test Method Line No.", AITLogEntry."Test Suite Code"); + + if AITTestMethodLineForLogEntry."Codeunit ID" <> AITLogEntry."Codeunit ID" then + Error(TestSuiteChangedErr); + + AITTestRunInputHandler.SetInput(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code"); + + BindSubscription(AITTestRunInputHandler); + RunAITestLine(AITTestMethodLineForLogEntry, false); + UnbindSubscription(AITTestRunInputHandler); + + AITTestSuite.Get(AITTestMethodLineForLogEntry."Test Suite Code"); + exit(AITTestSuite.Version); end; internal procedure RunAITestLine(AITTestMethodLine: Record "AIT Test Method Line"; IsExecutedFromTestSuiteHeader: Boolean) @@ -131,10 +154,24 @@ codeunit 149034 "AIT Test Suite Mgt." AITTestMethodLine.CalcFields("No. of Tests Executed", "No. of Tests Passed", "Total Duration (ms)"); TelemetryCustomDimensions := GetFeatureUsedInsights(EmptyGuid, AITTestSuite.Version, AITTestMethodLine."No. of Tests Executed", AITTestMethodLine."No. of Tests Passed", AITTestMethodLine."Total Duration (ms)"); FeatureTelemetry.LogUptake('0000NEY', GetFeatureName(), Enum::"Feature Uptake Status"::Used, TelemetryCustomDimensions); + LogRunHistory(AITTestSuite.Code, AITTestSuite.Version, AITTestSuite.Tag); end; end; end; + local procedure LogRunHistory(Code: Code[10]; Version: Integer; Tag: Text[20]) + var + AITRunHistory: Record "AIT Run History"; + AITTestContext: Codeunit "AIT Test Context"; + begin + AITRunHistory."Test Suite Code" := Code; + AITRunHistory.Version := Version; + AITRunHistory.Tag := Tag; + AITRunHistory.Insert(); + + AITTestContext.OnAfterRunComplete(Code, Version, Tag); + end; + local procedure ValidateAITestSuite(AITTestSuite: Record "AIT Test Suite") var AITTestMethodLine: Record "AIT Test Method Line"; @@ -355,6 +392,9 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."Procedure Name" := CurrentTestMethodLine.Function; AITLogEntry."Tokens Consumed" := AITTestRunIteration.GetAITokenUsedByLastTestMethodLine(); + AITLogEntry."No. of Turns" := AITTestRunIteration.GetNumberOfTurnsForLastTestMethodLine(); + AITLogEntry."No. of Turns Passed" := AITTestRunIteration.GetNumberOfTurnsPassedForLastTestMethodLine(); + AITLogEntry."Test Method Line Accuracy" := AITTestRunIteration.GetAccuracyForLastTestMethodLine(); AITLogEntry.Insert(true); Commit(); @@ -371,6 +411,39 @@ codeunit 149034 "AIT Test Suite Mgt." TelemetryCustomDimensions.Add('TotalDurationInMs', Format(TotalDurationInMs)); end; + internal procedure GetEvaluationSetupText(TestSuiteCode: Code[10]; LineNo: Integer): Text + var + AITEvaluators: Record "AIT Evaluator"; + TextBuilder: TextBuilder; + begin + AITEvaluators.SetRange("Test Suite Code", TestSuiteCode); + AITEvaluators.SetRange("Test Method Line", LineNo); + + if AITEvaluators.IsEmpty() then + exit(NoEvaluatorsLbl); + + TextBuilder.Append('{'); + if AITEvaluators.FindSet() then begin + repeat + TextBuilder.Append('"' + AITEvaluators.Evaluator + '",'); + until AITEvaluators.Next() = 0; + TextBuilder.Remove(TextBuilder.Length(), 1); // Remove the last comma + end; + TextBuilder.Append('}'); + exit(TextBuilder.ToText()); + end; + + internal procedure GetTurnsAsText(var AITTestMethodLine: Record "AIT Test Method Line"): Text + begin + AITTestMethodLine.CalcFields("No. of Turns Passed", "No. of Turns"); + exit(StrSubstNo(TurnsLbl, AITTestMethodLine."No. of Turns Passed", AITTestMethodLine."No. of Turns")); + end; + + internal procedure GetTurnsAsText(var AITLogEntry: Record "AIT Log Entry"): Text + begin + exit(StrSubstNo(TurnsLbl, AITLogEntry."No. of Turns Passed", AITLogEntry."No. of Turns")); + end; + internal procedure GetAvgDuration(AITTestMethodLine: Record "AIT Test Method Line"): Integer begin if AITTestMethodLine."No. of Tests Executed" = 0 then @@ -444,6 +517,8 @@ codeunit 149034 "AIT Test Suite Mgt." local procedure DeleteLinesOnDeleteAITTestSuite(var Rec: Record "AIT Test Suite"; RunTrigger: Boolean) var AITTestMethodLine: Record "AIT Test Method Line"; + AITEvaluator: Record "AIT Evaluator"; + AITColumnMapping: Record "AIT Column Mapping"; AITLogEntry: Record "AIT Log Entry"; AITRunHistory: Record "AIT Run History"; begin @@ -453,6 +528,12 @@ codeunit 149034 "AIT Test Suite Mgt." AITTestMethodLine.SetRange("Test Suite Code", Rec."Code"); AITTestMethodLine.DeleteAll(true); + AITEvaluator.SetRange("Test Suite Code", Rec."Code"); + AITEvaluator.DeleteAll(true); + + AITColumnMapping.SetRange("Test Suite Code", Rec."Code"); + AITColumnMapping.DeleteAll(true); + AITLogEntry.SetRange("Test Suite Code", Rec."Code"); AITLogEntry.DeleteAll(true); @@ -480,10 +561,20 @@ codeunit 149034 "AIT Test Suite Mgt." local procedure DeleteLogEntriesOnDeleteAITTestMethodLine(var Rec: Record "AIT Test Method Line"; RunTrigger: Boolean) var AITLogEntry: Record "AIT Log Entry"; + AITEvaluator: Record "AIT Evaluator"; + AITColumnMapping: Record "AIT Column Mapping"; begin if Rec.IsTemporary() then exit; + AITEvaluator.SetRange("Test Suite Code", Rec."Test Suite Code"); + AITEvaluator.SetRange("Test Method Line", Rec."Line No."); + AITEvaluator.DeleteAll(true); + + AITColumnMapping.SetRange("Test Suite Code", Rec."Test Suite Code"); + AITColumnMapping.SetRange("Test Method Line", Rec."Line No."); + AITColumnMapping.DeleteAll(true); + AITLogEntry.SetRange("Test Suite Code", Rec."Test Suite Code"); AITLogEntry.SetRange("Test Method Line No.", Rec."Line No."); AITLogEntry.DeleteAll(true);