diff --git a/examples/demo/9-react.pdl b/examples/demo/9-react.pdl index 13621edcd..4225ed076 100644 --- a/examples/demo/9-react.pdl +++ b/examples/demo/9-react.pdl @@ -1,31 +1,33 @@ description: Demonstrating a React program defs: + Calc: + description: Calculator function + function: + expr: + type: string + description: Arithmetic expression to calculate + return: + lang: python + code: result = ${ expr } + Search: + description: Wikipedia search + function: + topic: + type: string + description: Topic to search + return: + lang: python + code: | + import warnings, wikipedia + warnings.simplefilter("ignore") + try: + result = wikipedia.summary("${ topic }") + except wikipedia.WikipediaException as e: + result = str(e) tools: data: - - type: function - function: - name: Calc - description: Calculator function - parameters: - type: object - properties: - expr: - type: string - description: Arithmetic expression to calculate - required: - - expr - - type: function - function: - name: Search - description: Wikipedia search - parameters: - type: object - properties: - topic: - type: string - description: Topic to search - required: - - topic + - ${ Calc.signature } + - ${ Search.signature } text: - role: system text: You are Granite, developed by IBM. You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request. @@ -77,19 +79,17 @@ text: - "\nObservation: " - match: ${ action[0].name } with: - - case: 'Search' + - case: Search then: - lang: python - code: | - import warnings, wikipedia - warnings.simplefilter("ignore") - try: - result = wikipedia.summary("${ action[0].arguments.topic }") - except wikipedia.WikipediaException as e: - result = str(e) - - case: 'Calc' + call: ${ Search } + args: + topic: ${ action[0].arguments.topic } + - case: Calc then: - lang: python - code: result = ${ action[0].arguments.expr } + call: ${ Calc } + args: + expr: ${ action[0].arguments.expr } + - case: Finish + then: ${ action[0].arguments.topic } - "\n" - until: ${ action[0].name == "Finish" } \ No newline at end of file + until: ${ action[0].name == "Finish" } diff --git a/examples/demos/react.pdl b/examples/demos/react.pdl index 43b2e2552..4225ed076 100644 --- a/examples/demos/react.pdl +++ b/examples/demos/react.pdl @@ -1,30 +1,33 @@ +description: Demonstrating a React program defs: + Calc: + description: Calculator function + function: + expr: + type: string + description: Arithmetic expression to calculate + return: + lang: python + code: result = ${ expr } + Search: + description: Wikipedia search + function: + topic: + type: string + description: Topic to search + return: + lang: python + code: | + import warnings, wikipedia + warnings.simplefilter("ignore") + try: + result = wikipedia.summary("${ topic }") + except wikipedia.WikipediaException as e: + result = str(e) tools: data: - - type: function - function: - name: Calc - description: Calculator function - parameters: - type: object - properties: - expr: - type: string - description: Arithmetic expression to calculate - required: - - expr - - type: function - function: - name: Search - description: Wikipedia search - parameters: - type: object - properties: - topic: - type: string - description: Topic to search - required: - - topic + - ${ Calc.signature } + - ${ Search.signature } text: - role: system text: You are Granite, developed by IBM. You are a helpful AI assistant with access to the following tools. When a tool is required to answer the user's query, respond with <|tool_call|> followed by a JSON list of tools used. If a tool does not exist in the provided list of tools, notify the user that you do not have the ability to fulfill the request. @@ -76,19 +79,17 @@ text: - "\nObservation: " - match: ${ action[0].name } with: - - case: 'Search' + - case: Search then: - lang: python - code: | - import warnings, wikipedia - warnings.simplefilter("ignore") - try: - result = wikipedia.summary("${ action[0].arguments.topic }") - except wikipedia.WikipediaException as e: - result = str(e) - - case: 'Calc' + call: ${ Search } + args: + topic: ${ action[0].arguments.topic } + - case: Calc then: - lang: python - code: result = ${ action[0].arguments.expr } + call: ${ Calc } + args: + expr: ${ action[0].arguments.expr } + - case: Finish + then: ${ action[0].arguments.topic } - "\n" - until: ${ action[0].name == "Finish" } \ No newline at end of file + until: ${ action[0].name == "Finish" } diff --git a/tests/results/examples/demo/9-react.0.result b/tests/results/examples/demo/9-react.0.result index 679bcf2cd..040d4961f 100644 --- a/tests/results/examples/demo/9-react.0.result +++ b/tests/results/examples/demo/9-react.0.result @@ -11,4 +11,4 @@ Action: Observation: 460 Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.1.result b/tests/results/examples/demo/9-react.1.result index a670f1757..d544ecc82 100644 --- a/tests/results/examples/demo/9-react.1.result +++ b/tests/results/examples/demo/9-react.1.result @@ -21,4 +21,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.2.result b/tests/results/examples/demo/9-react.2.result index 14c95d61f..592094826 100644 --- a/tests/results/examples/demo/9-react.2.result +++ b/tests/results/examples/demo/9-react.2.result @@ -23,4 +23,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.3.result b/tests/results/examples/demo/9-react.3.result index a670f1757..d544ecc82 100644 --- a/tests/results/examples/demo/9-react.3.result +++ b/tests/results/examples/demo/9-react.3.result @@ -21,4 +21,4 @@ Action: Observation: 460 In the year 2025, it would be approximately 460 years ago that Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.4.result b/tests/results/examples/demo/9-react.4.result index 679bcf2cd..040d4961f 100644 --- a/tests/results/examples/demo/9-react.4.result +++ b/tests/results/examples/demo/9-react.4.result @@ -11,4 +11,4 @@ Action: Observation: 460 Henry Hudson was born around 1565, so he was approximately 460 years ago.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.5.result b/tests/results/examples/demo/9-react.5.result index 279b584af..4f4a4e534 100644 --- a/tests/results/examples/demo/9-react.5.result +++ b/tests/results/examples/demo/9-react.5.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 Henry Hudson was born around 1565, which means he was born approximately 460 years ago as of 2025.Action: [{"name": "Finish", "arguments": {"topic": ""}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demo/9-react.6.result b/tests/results/examples/demo/9-react.6.result index ebce142c0..469ab46c5 100644 --- a/tests/results/examples/demo/9-react.6.result +++ b/tests/results/examples/demo/9-react.6.result @@ -12,4 +12,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 It has been approximately 460 years since Henry Hudson was born.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 diff --git a/tests/results/examples/demo/9-react.7.result b/tests/results/examples/demo/9-react.7.result new file mode 100644 index 000000000..f9520b25c --- /dev/null +++ b/tests/results/examples/demo/9-react.7.result @@ -0,0 +1,13 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 diff --git a/tests/results/examples/demo/9-react.8.result b/tests/results/examples/demo/9-react.8.result new file mode 100644 index 000000000..a715aadda --- /dev/null +++ b/tests/results/examples/demo/9-react.8.result @@ -0,0 +1,15 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 1565, so in 2025, it has been approximately 460 years since his birth.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 diff --git a/tests/results/examples/demos/react.0.result b/tests/results/examples/demos/react.0.result index 279b584af..4f4a4e534 100644 --- a/tests/results/examples/demos/react.0.result +++ b/tests/results/examples/demos/react.0.result @@ -10,4 +10,4 @@ Henry Hudson was born around 1565. As of 2025, it has been approximately 470 yea Observation: 460 Henry Hudson was born around 1565, which means he was born approximately 460 years ago as of 2025.Action: [{"name": "Finish", "arguments": {"topic": ""}}] -Observation: \ No newline at end of file +Observation: 460 \ No newline at end of file diff --git a/tests/results/examples/demos/react.1.result b/tests/results/examples/demos/react.1.result index ec6f7668e..b171b1896 100644 --- a/tests/results/examples/demos/react.1.result +++ b/tests/results/examples/demos/react.1.result @@ -1,15 +1,16 @@ - How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. -Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to search who discovered the Hudson River. Action: [{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. - - -Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +Thought: Henry Hudson was born around 1565. To find out how many years ago he was born from the year 2025, we subtract 2025 from 1565. +Calculation: 2025 - 1565 = 460 +Action: [{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] Observation: 460 -It has been approximately 460 years since Henry Hudson was born.Action: +Henry Hudson was born around 460 years before 2025.Action: [{"name": "Finish", "arguments": {"topic": "460"}}] -Observation: +Observation: 460 + diff --git a/tests/results/examples/demos/react.2.result b/tests/results/examples/demos/react.2.result new file mode 100644 index 000000000..f9520b25c --- /dev/null +++ b/tests/results/examples/demos/react.2.result @@ -0,0 +1,13 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 diff --git a/tests/results/examples/demos/react.3.result b/tests/results/examples/demos/react.3.result new file mode 100644 index 000000000..a4c21f3c9 --- /dev/null +++ b/tests/results/examples/demos/react.3.result @@ -0,0 +1,16 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to search who discovered the Hudson River. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. +Thought: Henry Hudson was born around 1565. To find out how many years ago he was born from the year 2025, we subtract 2025 from 1565. +Calculation: 2025 - 1565 = 460 +Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +Henry Hudson was born around 460 years before 2025.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: + diff --git a/tests/results/examples/demos/react.4.result b/tests/results/examples/demos/react.4.result new file mode 100644 index 000000000..469ab46c5 --- /dev/null +++ b/tests/results/examples/demos/react.4.result @@ -0,0 +1,15 @@ +How many years ago was the discoverer of the Hudson River born? Keep in mind we are in 2025. When searching for a birthday for a person, simply ask for the name of that person. +Thought: I need to find out who discovered the Hudson River and then calculate how many years ago they were born based on the current year, 2025. +Action: +[{"name": "Search", "arguments": {"topic": "Henry Hudson"}}] +Observation: Henry Hudson (c. 1565 – disappeared 23 June 1611) was an English sea explorer and navigator during the early 17th century, best known for his explorations of present-day Canada and parts of the Northeastern United States. +In 1607 and 1608, Hudson made two attempts on behalf of English merchants to find a rumoured Northeast Passage to Cathay via a route above the Arctic Circle. In 1609, he landed in North America on behalf of the Dutch East India Company and explored the region around the modern New York metropolitan area. Looking for a Northwest Passage to Asia on his ship Halve Maen ("Half Moon"), he sailed up the Hudson River, which was later named after him, and thereby laid the foundation for Dutch colonization of the region. His contributions to the exploration of the New World were significant and lasting. His voyages helped to establish European contact with the native peoples of North America and contributed to the development of trade and commerce. +On his final expedition, while still searching for the Northwest Passage, Hudson became the first European to see Hudson Strait and the immense Hudson Bay. In 1611, after wintering on the shore of James Bay, Hudson wanted to press on to the west, but most of his crew mutinied. The mutineers cast Hudson, his son, and six others adrift; what then happened to the Hudsons and their companions is unknown. + + +Henry Hudson was born around 1565. As of 2025, it has been approximately 470 years since Henry Hudson's birth.Action: +[{"name": "Calc", "arguments": {"expr": "2025 - 1565"}}] +Observation: 460 +It has been approximately 460 years since Henry Hudson was born.Action: +[{"name": "Finish", "arguments": {"topic": "460"}}] +Observation: 460 diff --git a/tests/test_examples_run.py b/tests/test_examples_run.py index fd62b39ff..b0b724f34 100644 --- a/tests/test_examples_run.py +++ b/tests/test_examples_run.py @@ -102,8 +102,8 @@ class FailedResults: """ wrong_results: Dict[str, str] = field(default_factory=lambda: {}) - unexpected_parse_error: List[str] = field(default_factory=lambda: []) - unexpected_runtime_error: List[str] = field(default_factory=lambda: []) + unexpected_parse_error: Dict[str, str] = field(default_factory=lambda: {}) + unexpected_runtime_error: Dict[str, str] = field(default_factory=lambda: {}) # pylint: disable=too-many-instance-attributes @@ -229,9 +229,11 @@ def __execute_file(self, pdl_file_name: str) -> None: exec_result.result = str(output["result"]) exec_result.error_code = ExecutionErrorCode.NO_ERROR pdl.write_trace("/dev/null", output["trace"]) - except PDLParseError: + except PDLParseError as exc: + exec_result.result = str(exc) exec_result.error_code = ExecutionErrorCode.PARSE_ERROR - except Exception: + except Exception as exc: + exec_result.result = str(exc) exec_result.error_code = ExecutionErrorCode.RUNTIME_ERROR self.execution_results[pdl_file_name] = exec_result @@ -252,8 +254,8 @@ def validate_expected_and_actual(self) -> None: """ wrong_result: Dict[str, str] = {} - unexpected_parse_error: List[str] = [] - unexpected_runtime_error: List[str] = [] + unexpected_parse_error: Dict[str, str] = {} + unexpected_runtime_error: Dict[str, str] = {} for file in self.check: if file not in self.skip: @@ -264,9 +266,9 @@ def validate_expected_and_actual(self) -> None: if not match: # Check if actual results caused any error if actual_result.error_code == ExecutionErrorCode.PARSE_ERROR: - unexpected_parse_error.append(file) + unexpected_parse_error[file] = str(actual_result.result) elif actual_result.error_code == ExecutionErrorCode.RUNTIME_ERROR: - unexpected_runtime_error.append(file) + unexpected_runtime_error[file] = str(actual_result.result) # If no error, then the results are wrong else: if actual_result.result is not None: @@ -315,6 +317,26 @@ def test_example_runs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> if background.update_results: background.write_results() + # Print the unexpected parse errors + for file, actual in background.failed_results.unexpected_parse_error.items(): + print( + "\n============================================================================" + ) + print(f"File that produced unexpected parse error: {file}") + print( + f"Error message:\n--------------------------------------------------------------\n{actual}\n-------------------------------------------------------------" + ) + + # Print the runtime errors + for file, actual in background.failed_results.unexpected_runtime_error.items(): + print( + "\n============================================================================" + ) + print(f"File that produced unexpected runtime error: {file}") + print( + f"Error message:\n--------------------------------------------------------------\n{actual}\n-------------------------------------------------------------" + ) + # Print the actual results for wrong results for file, actual in background.failed_results.wrong_results.items(): print( diff --git a/tests/test_examples_run.yaml b/tests/test_examples_run.yaml index 9518039fa..78ab19b4e 100644 --- a/tests/test_examples_run.yaml +++ b/tests/test_examples_run.yaml @@ -1,6 +1,7 @@ update_results: false check: [] skip: + - examples/demos/react.pdl - examples/cldk/cldk-assistant.pdl - examples/gsm8k/gsm8.pdl - examples/gsm8k/gsm8k-plan.pdl