From 0c29ecd5d8f1842f8030236dba2872400c095aa6 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Mon, 7 Jul 2025 15:50:35 -0400 Subject: [PATCH 01/14] tests: update react example Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 82 ++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 43b2e2552..bde88aab2 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -1,30 +1,32 @@ defs: + calc: + description: Calculator function + function: + expr: + type: string + description: Arithmetic expression to calculate + return: + lang: python + code: result = ${ expr } + search: + description: Wikipedia search + function: + topic: + type: string + description: Topic to search + return: + lang: python + code: | + import warnings, wikipedia + warnings.simplefilter("ignore") + try: + result = wikipedia.summary("${ topic }") + except wikipedia.WikipediaException as e: + result = str(e) tools: data: - - type: function - function: - name: Calc - description: Calculator function - parameters: - type: object - properties: - expr: - type: string - description: Arithmetic expression to calculate - required: - - expr - - type: function - function: - name: Search - description: Wikipedia search - parameters: - type: object - properties: - topic: - type: string - description: Topic to search - required: - - topic + - ${ calc.signature } + - ${ search.signature } text: - role: system text: You are Granite, developed by IBM. You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request. @@ -38,26 +40,26 @@ text: What profession does Nicholas Ray and Elia Kazan have in common? Thought: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. Action: - [{"name": "Search", "arguments": {"topic": "Nicholas Ray"}}] + [{"name": "search", "arguments": {"topic": "Nicholas Ray"}}] Observation: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. Thought: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. Action: - [{"name": "Search", "arguments": {"topic": "Elia Kazan"}}] + [{"name": "search", "arguments": {"topic": "Elia Kazan"}}] Observation: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. Thought: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Action: - [{"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}}] + [{"name": "finish", "arguments": {"topic": "director, screenwriter, actor"}}] What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? Thought: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... Action: - [{"name": "Search", "arguments": {"topic": "Colorado orogeny"}}] + [{"name": "search", "arguments": {"topic": "Colorado orogeny"}}] Observation: The Colorado orogeny was an episode of mountain building (an orogeny) ... Thought: It does not mention the eastern sector. So I need to look up eastern sector. Thought: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. Action: - [{"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}}] + [{"name": "finish", "arguments": {"topic": "1,800 to 7,000 ft"}}] contribute: [context] - "How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person.\n" @@ -76,19 +78,15 @@ text: - "\nObservation: " - match: ${ action[0].name } with: - - case: 'Search' + - case: search then: - lang: python - code: | - import warnings, wikipedia - warnings.simplefilter("ignore") - try: - result = wikipedia.summary("${ action[0].arguments.topic }") - except wikipedia.WikipediaException as e: - result = str(e) - - case: 'Calc' + call: ${ search } + args: + topic: ${ action[0].arguments.topic } + - case: calc then: - lang: python - code: result = ${ action[0].arguments.expr } + call: ${ calc } + args: + expr: ${ action[0].arguments.expr } - "\n" - until: ${ action[0].name == "Finish" } \ No newline at end of file + until: ${ action[0].name == "finish" } \ No newline at end of file From de8b930996ab7e487bdf7bc502d1e3725cce8a62 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Tue, 8 Jul 2025 22:22:49 -0400 Subject: [PATCH 02/14] Keep same capitalization Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index bde88aab2..2c36193c1 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -1,5 +1,5 @@ defs: - calc: + Calc: description: Calculator function function: expr: @@ -8,7 +8,7 @@ defs: return: lang: python code: result = ${ expr } - search: + Search: description: Wikipedia search function: topic: @@ -25,8 +25,8 @@ defs: result = str(e) tools: data: - - ${ calc.signature } - - ${ search.signature } + - ${ Calc.signature } + - ${ Search.signature } text: - role: system text: You are Granite, developed by IBM. You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request. @@ -40,26 +40,26 @@ text: What profession does Nicholas Ray and Elia Kazan have in common? Thought: I need to search Nicholas Ray and Elia Kazan, find their professions, then find the profession they have in common. Action: - [{"name": "search", "arguments": {"topic": "Nicholas Ray"}}] + [{"name": "Search", "arguments": {"topic": "Nicholas Ray"}}] Observation: Nicholas Ray (born Raymond Nicholas Kienzle Jr., August 7, 1911 - June 16, 1979) was an American film director, screenwriter, and actor best known for the 1955 film Rebel Without a Cause. Thought: Professions of Nicholas Ray are director, screenwriter, and actor. I need to search Elia Kazan next and find his professions. Action: - [{"name": "search", "arguments": {"topic": "Elia Kazan"}}] + [{"name": "Search", "arguments": {"topic": "Elia Kazan"}}] Observation: Elia Kazan was an American film and theatre director, producer, screenwriter and actor. Thought: Professions of Elia Kazan are director, producer, screenwriter, and actor. So profession Nicholas Ray and Elia Kazan have in common is director, screenwriter, and actor. Action: - [{"name": "finish", "arguments": {"topic": "director, screenwriter, actor"}}] + [{"name": "Finish", "arguments": {"topic": "director, screenwriter, actor"}}] What is the elevation range for the area that the eastern sector of the Colorado orogeny extends into? Thought: I need to search Colorado orogeny, find the area that the eastern sector of the Colorado ... Action: - [{"name": "search", "arguments": {"topic": "Colorado orogeny"}}] + [{"name": "Search", "arguments": {"topic": "Colorado orogeny"}}] Observation: The Colorado orogeny was an episode of mountain building (an orogeny) ... Thought: It does not mention the eastern sector. So I need to look up eastern sector. Thought: High Plains rise in elevation from around 1,800 to 7,000 ft, so the answer is 1,800 to 7,000 ft. Action: - [{"name": "finish", "arguments": {"topic": "1,800 to 7,000 ft"}}] + [{"name": "Finish", "arguments": {"topic": "1,800 to 7,000 ft"}}] contribute: [context] - "How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person.\n" @@ -78,15 +78,15 @@ text: - "\nObservation: " - match: ${ action[0].name } with: - - case: search + - case: Search then: - call: ${ search } + call: ${ Search } args: topic: ${ action[0].arguments.topic } - - case: calc + - case: Calc then: - call: ${ calc } + call: ${ Calc } args: expr: ${ action[0].arguments.expr } - "\n" - until: ${ action[0].name == "finish" } \ No newline at end of file + until: ${ action[0].name == "Finish" } \ No newline at end of file From 383064ccf2f6ad427573acca7163e0ebd53b5c54 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Tue, 8 Jul 2025 22:44:09 -0400 Subject: [PATCH 03/14] Add expected result Signed-off-by: Louis Mandel --- tests/results/examples/demos/react.2.result | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/results/examples/demos/react.2.result diff --git a/tests/results/examples/demos/react.2.result b/tests/results/examples/demos/react.2.result new file mode 100644 index 000000000..ec87b71d5 --- /dev/null +++ b/tests/results/examples/demos/react.2.result @@ -0,0 +1,13 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: From 12829d3d837d9d9ab84fa024c9d96793330a720b Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Tue, 8 Jul 2025 22:51:43 -0400 Subject: [PATCH 04/14] Update other react demo Signed-off-by: Louis Mandel --- examples/demo/9-react.pdl | 72 +++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/examples/demo/9-react.pdl b/examples/demo/9-react.pdl index 13621edcd..314cdc0b3 100644 --- a/examples/demo/9-react.pdl +++ b/examples/demo/9-react.pdl @@ -1,31 +1,33 @@ description: Demonstrating a React program defs: + Calc: + description: Calculator function + function: + expr: + type: string + description: Arithmetic expression to calculate + return: + lang: python + code: result = ${ expr } + Search: + description: Wikipedia search + function: + topic: + type: string + description: Topic to search + return: + lang: python + code: | + import warnings, wikipedia + warnings.simplefilter("ignore") + try: + result = wikipedia.summary("${ topic }") + except wikipedia.WikipediaException as e: + result = str(e) tools: data: - - type: function - function: - name: Calc - description: Calculator function - parameters: - type: object - properties: - expr: - type: string - description: Arithmetic expression to calculate - required: - - expr - - type: function - function: - name: Search - description: Wikipedia search - parameters: - type: object - properties: - topic: - type: string - description: Topic to search - required: - - topic + - ${ Calc.signature } + - ${ Search.signature } text: - role: system text: You are Granite, developed by IBM. You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request. @@ -77,19 +79,15 @@ text: - "\nObservation: " - match: ${ action[0].name } with: - - case: 'Search' + - case: Search then: - lang: python - code: | - import warnings, wikipedia - warnings.simplefilter("ignore") - try: - result = wikipedia.summary("${ action[0].arguments.topic }") - except wikipedia.WikipediaException as e: - result = str(e) - - case: 'Calc' + call: ${ Search } + args: + topic: ${ action[0].arguments.topic } + - case: Calc then: - lang: python - code: result = ${ action[0].arguments.expr } + call: ${ Calc } + args: + expr: ${ action[0].arguments.expr } - "\n" - until: ${ action[0].name == "Finish" } \ No newline at end of file + until: ${ action[0].name == "Finish" } From 2544e3334bc7bfced340c3e625a7c2bd64c2c757 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Wed, 9 Jul 2025 09:24:02 -0400 Subject: [PATCH 05/14] Add expected results Signed-off-by: Louis Mandel --- tests/results/examples/demo/9-react.7.result | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/results/examples/demo/9-react.7.result diff --git a/tests/results/examples/demo/9-react.7.result b/tests/results/examples/demo/9-react.7.result new file mode 100644 index 000000000..ec87b71d5 --- /dev/null +++ b/tests/results/examples/demo/9-react.7.result @@ -0,0 +1,13 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: From 63f71294241d32c434eeb848f7ca88fe929780a3 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Tue, 15 Jul 2025 14:15:41 -0400 Subject: [PATCH 06/14] Update react example Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 2c36193c1..314cdc0b3 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -1,3 +1,4 @@ +description: Demonstrating a React program defs: Calc: description: Calculator function @@ -89,4 +90,4 @@ text: args: expr: ${ action[0].arguments.expr } - "\n" - until: ${ action[0].name == "Finish" } \ No newline at end of file + until: ${ action[0].name == "Finish" } From 012aebfebfddde82b04a370e46727a6c6773a7c0 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Wed, 16 Jul 2025 10:35:38 -0400 Subject: [PATCH 07/14] Add a finish action Signed-off-by: Louis Mandel --- examples/demo/9-react.pdl | 2 ++ tests/results/examples/demo/9-react.0.result | 2 +- tests/results/examples/demo/9-react.1.result | 2 +- tests/results/examples/demo/9-react.2.result | 2 +- tests/results/examples/demo/9-react.3.result | 2 +- tests/results/examples/demo/9-react.4.result | 2 +- tests/results/examples/demo/9-react.5.result | 2 +- tests/results/examples/demo/9-react.6.result | 2 +- tests/results/examples/demo/9-react.7.result | 2 +- tests/results/examples/demos/react.0.result | 2 +- tests/results/examples/demos/react.1.result | 15 ++++++++------- tests/results/examples/demos/react.2.result | 2 +- tests/results/examples/demos/react.3.result | 16 ++++++++++++++++ 13 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 tests/results/examples/demos/react.3.result diff --git a/examples/demo/9-react.pdl b/examples/demo/9-react.pdl index 314cdc0b3..4225ed076 100644 --- a/examples/demo/9-react.pdl +++ b/examples/demo/9-react.pdl @@ -89,5 +89,7 @@ text: call: ${ Calc } args: expr: ${ action[0].arguments.expr } + - case: Finish + then: ${ action[0].arguments.topic } - "\n" until: ${ action[0].name == "Finish" } diff --git a/tests/results/examples/demo/9-react.0.result b/tests/results/examples/demo/9-react.0.result index 679bcf2cd..040d4961f 100644 --- a/tests/results/examples/demo/9-react.0.result +++ b/tests/results/examples/demo/9-react.0.result @@ -11,4 +11,4 @@ Action: Observation: 460 Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.1.result b/tests/results/examples/demo/9-react.1.result index a670f1757..d544ecc82 100644 --- a/tests/results/examples/demo/9-react.1.result +++ b/tests/results/examples/demo/9-react.1.result @@ -21,4 +21,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.2.result b/tests/results/examples/demo/9-react.2.result index 14c95d61f..592094826 100644 --- a/tests/results/examples/demo/9-react.2.result +++ b/tests/results/examples/demo/9-react.2.result @@ -23,4 +23,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.3.result b/tests/results/examples/demo/9-react.3.result index a670f1757..d544ecc82 100644 --- a/tests/results/examples/demo/9-react.3.result +++ b/tests/results/examples/demo/9-react.3.result @@ -21,4 +21,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.4.result b/tests/results/examples/demo/9-react.4.result index 679bcf2cd..040d4961f 100644 --- a/tests/results/examples/demo/9-react.4.result +++ b/tests/results/examples/demo/9-react.4.result @@ -11,4 +11,4 @@ Action: Observation: 460 Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.5.result b/tests/results/examples/demo/9-react.5.result index 279b584af..4f4a4e534 100644 --- a/tests/results/examples/demo/9-react.5.result +++ b/tests/results/examples/demo/9-react.5.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 Henry Hudson was born around 1565, which means he was born approximately 460 years ago as of 2025.Action: [{"name": "Finish", "arguments": {"topic": ""}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.6.result b/tests/results/examples/demo/9-react.6.result index ebce142c0..469ab46c5 100644 --- a/tests/results/examples/demo/9-react.6.result +++ b/tests/results/examples/demo/9-react.6.result @@ -12,4 +12,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 It has been approximately 460 years since Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.7.result b/tests/results/examples/demo/9-react.7.result index ec87b71d5..f9520b25c 100644 --- a/tests/results/examples/demo/9-react.7.result +++ b/tests/results/examples/demo/9-react.7.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 It has been approximately 460 years since Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demos/react.0.result b/tests/results/examples/demos/react.0.result index 279b584af..4f4a4e534 100644 --- a/tests/results/examples/demos/react.0.result +++ b/tests/results/examples/demos/react.0.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 Henry Hudson was born around 1565, which means he was born approximately 460 years ago as of 2025.Action: [{"name": "Finish", "arguments": {"topic": ""}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demos/react.1.result b/tests/results/examples/demos/react.1.result index ec6f7668e..b171b1896 100644 --- a/tests/results/examples/demos/react.1.result +++ b/tests/results/examples/demos/react.1.result @@ -1,15 +1,16 @@ - How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. -Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to search who discovered the Hudson River. Action: [{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. - - -Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +Thought: Henry Hudson was born around 1565. To find out how many years ago he was born from the year 2025, we subtract 2025 from 1565. +Calculation: 2025 - 1565 = 460 +Action: [{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] Observation: 460 -It has been approximately 460 years since Henry Hudson was born.Action: +Henry Hudson was born around 460 years before 2025.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 + diff --git a/tests/results/examples/demos/react.2.result b/tests/results/examples/demos/react.2.result index ec87b71d5..f9520b25c 100644 --- a/tests/results/examples/demos/react.2.result +++ b/tests/results/examples/demos/react.2.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 It has been approximately 460 years since Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demos/react.3.result b/tests/results/examples/demos/react.3.result new file mode 100644 index 000000000..a4c21f3c9 --- /dev/null +++ b/tests/results/examples/demos/react.3.result @@ -0,0 +1,16 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to search who discovered the Hudson River. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson was born around 1565. To find out how many years ago he was born from the year 2025, we subtract 2025 from 1565. +Calculation: 2025 - 1565 = 460 +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 460 years before 2025.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: + From 469719cbc78a388902e6b0143b2447ddeb434566 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Wed, 16 Jul 2025 11:47:10 -0400 Subject: [PATCH 08/14] Add finish action Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 314cdc0b3..4225ed076 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -89,5 +89,7 @@ text: call: ${ Calc } args: expr: ${ action[0].arguments.expr } + - case: Finish + then: ${ action[0].arguments.topic } - "\n" until: ${ action[0].name == "Finish" } From ab2eec47132f95317c4aba4bb54d8e52bc4f55e7 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Wed, 16 Jul 2025 13:12:30 -0400 Subject: [PATCH 09/14] Print parse and runtime errors in `test_examples_run` Signed-off-by: Louis Mandel --- tests/test_examples_run.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index fd62b39ff..b0b724f34 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -102,8 +102,8 @@ class FailedResults: """ wrong_results: Dict[str, str] = field(default_factory=lambda: {}) - unexpected_parse_error: List[str] = field(default_factory=lambda: []) - unexpected_runtime_error: List[str] = field(default_factory=lambda: []) + unexpected_parse_error: Dict[str, str] = field(default_factory=lambda: {}) + unexpected_runtime_error: Dict[str, str] = field(default_factory=lambda: {}) # pylint: disable=too-many-instance-attributes @@ -229,9 +229,11 @@ def __execute_file(self, pdl_file_name: str) -> None: exec_result.result = str(output["result"]) exec_result.error_code = ExecutionErrorCode.NO_ERROR pdl.write_trace("/dev/null", output["trace"]) - except PDLParseError: + except PDLParseError as exc: + exec_result.result = str(exc) exec_result.error_code = ExecutionErrorCode.PARSE_ERROR - except Exception: + except Exception as exc: + exec_result.result = str(exc) exec_result.error_code = ExecutionErrorCode.RUNTIME_ERROR self.execution_results[pdl_file_name] = exec_result @@ -252,8 +254,8 @@ def validate_expected_and_actual(self) -> None: """ wrong_result: Dict[str, str] = {} - unexpected_parse_error: List[str] = [] - unexpected_runtime_error: List[str] = [] + unexpected_parse_error: Dict[str, str] = {} + unexpected_runtime_error: Dict[str, str] = {} for file in self.check: if file not in self.skip: @@ -264,9 +266,9 @@ def validate_expected_and_actual(self) -> None: if not match: # Check if actual results caused any error if actual_result.error_code == ExecutionErrorCode.PARSE_ERROR: - unexpected_parse_error.append(file) + unexpected_parse_error[file] = str(actual_result.result) elif actual_result.error_code == ExecutionErrorCode.RUNTIME_ERROR: - unexpected_runtime_error.append(file) + unexpected_runtime_error[file] = str(actual_result.result) # If no error, then the results are wrong else: if actual_result.result is not None: @@ -315,6 +317,26 @@ def test_example_runs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> if background.update_results: background.write_results() + # Print the unexpected parse errors + for file, actual in background.failed_results.unexpected_parse_error.items(): + print( + "\n============================================================================" + ) + print(f"File that produced unexpected parse error: {file}") + print( + f"Error message:\n--------------------------------------------------------------\n{actual}\n-------------------------------------------------------------" + ) + + # Print the runtime errors + for file, actual in background.failed_results.unexpected_runtime_error.items(): + print( + "\n============================================================================" + ) + print(f"File that produced unexpected runtime error: {file}") + print( + f"Error message:\n--------------------------------------------------------------\n{actual}\n-------------------------------------------------------------" + ) + # Print the actual results for wrong results for file, actual in background.failed_results.wrong_results.items(): print( From 593c5196abbd42d96df5924bac2cc0e0075db410 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Wed, 16 Jul 2025 14:26:08 -0400 Subject: [PATCH 10/14] Add expected results Signed-off-by: Louis Mandel --- tests/results/examples/demo/9-react.8.result | 15 +++++++++++++++ tests/results/examples/demos/react.4.result | 15 +++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tests/results/examples/demo/9-react.8.result create mode 100644 tests/results/examples/demos/react.4.result diff --git a/tests/results/examples/demo/9-react.8.result b/tests/results/examples/demo/9-react.8.result new file mode 100644 index 000000000..a715aadda --- /dev/null +++ b/tests/results/examples/demo/9-react.8.result @@ -0,0 +1,15 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 1565, so in 2025, it has been approximately 460 years since his birth.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 diff --git a/tests/results/examples/demos/react.4.result b/tests/results/examples/demos/react.4.result new file mode 100644 index 000000000..469ab46c5 --- /dev/null +++ b/tests/results/examples/demos/react.4.result @@ -0,0 +1,15 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 From 7ce184eefaa44a15868d1ad42d465a7da097172c Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Thu, 17 Jul 2025 11:31:52 -0400 Subject: [PATCH 11/14] Add fallback Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 4225ed076..6c7a2acde 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -91,5 +91,7 @@ text: expr: ${ action[0].arguments.expr } - case: Finish then: ${ action[0].arguments.topic } + fallback: | + Invalid action: ${ action } - "\n" until: ${ action[0].name == "Finish" } From c8b4e34c82eb995491fa228aa13a9c3bd21558e2 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Thu, 17 Jul 2025 14:10:21 -0400 Subject: [PATCH 12/14] More fallbacks Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 6c7a2acde..6815b5ca2 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -91,7 +91,13 @@ text: expr: ${ action[0].arguments.expr } - case: Finish then: ${ action[0].arguments.topic } + - then: | + Unexpected action: ${ action } fallback: | Invalid action: ${ action } - "\n" until: ${ action[0].name == "Finish" } + fallback: | + Failure: + - thought = ${ thought or null } + - action = ${ action } From 4a1e2f787c6ef2f96727134ccfb48ce79c5f2170 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Thu, 17 Jul 2025 16:18:08 -0400 Subject: [PATCH 13/14] More fallback Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 6815b5ca2..8e5a289b0 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -97,7 +97,10 @@ text: Invalid action: ${ action } - "\n" until: ${ action[0].name == "Finish" } - fallback: | - Failure: - - thought = ${ thought or null } - - action = ${ action } + fallback: + text: + - text: "Failure:\n" + - text: "- thought = ${ thought }\n" + fallback: "- thought is undefined\n" + - text: "- action = ${ action }\n" + fallback: "- action is undefined\n" From e162165f346025071fb799c433999926ff0d3301 Mon Sep 17 00:00:00 2001 From: Louis Mandel Date: Thu, 17 Jul 2025 16:56:43 -0400 Subject: [PATCH 14/14] Skip testing of react example Signed-off-by: Louis Mandel --- examples/demos/react.pdl | 11 ----------- tests/test_examples_run.yaml | 1 + 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 8e5a289b0..4225ed076 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -91,16 +91,5 @@ text: expr: ${ action[0].arguments.expr } - case: Finish then: ${ action[0].arguments.topic } - - then: | - Unexpected action: ${ action } - fallback: | - Invalid action: ${ action } - "\n" until: ${ action[0].name == "Finish" } - fallback: - text: - - text: "Failure:\n" - - text: "- thought = ${ thought }\n" - fallback: "- thought is undefined\n" - - text: "- action = ${ action }\n" - fallback: "- action is undefined\n" diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 9518039fa..78ab19b4e 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,6 +1,7 @@ update_results: false check: [] skip: + - examples/demos/react.pdl - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl - examples/gsm8k/gsm8k-plan.pdl