Add streaming support for PER agent #4384

nathaliellenaa · 2025-11-01T00:47:32Z

Description

Add streaming support for PER agent. It will stream the planner agent step and final LLM outputs.

Supported model:

OpenAI Chat Completion model
Bedrock Converse Stream model

Sample workflow:

// Register remote model
POST /_plugins/_ml/models/_register
{
    "name": "Bedrock converse stream",
    "function_name": "remote",
    "description": "bedrock claude model",
    "connector": {
        "name": "Amazon Bedrock Converse",
        "description": "Test connector for Amazon Bedrock Converse",
        "version": 1,
        "protocol": "aws_sigv4",
        "credential": {
            "access_key": "{{access_key}}",
            "secret_key": "{{secret_key}}",
            "session_token": "{{session_token}}"
        },
        "parameters": {
            "region": "{{aws_region}}",
            "service_name": "bedrock",
            "model": "us.anthropic.claude-3-7-sonnet-20250219-v1:0"
        },
        "actions": [
            {
                "action_type": "predict",
                "method": "POST",
                "headers": {
                    "content-type": "application/json"
                },
                "url": "https://bedrock-runtime.${parameters.region}.amazonaws.com/model/${parameters.model}/converse",
                "request_body": "{ \"system\": [{\"text\": \"${parameters.system_prompt}\"}], \"messages\": [${parameters._chat_history:-}{\"role\":\"user\",\"content\":[{\"text\":\"${parameters.prompt}\"}]}${parameters._interactions:-}]${parameters.tool_configs:-} }"
            }
        ]
    }
}

// Register PER agent
POST /_plugins/_ml/agents/_register
{
    "name": "My Plan Execute Reflect Agent",
    "type": "plan_execute_and_reflect",
    "description": "Agent for dynamic task planning and reasoning",
    "llm": {
        "model_id": "<model id from previous step>",
        "parameters": {
            "max_iteration": 5,
            "system_prompt": "You are a helpful assistant. You are able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics.\nIf the question is complex, you will split it into several smaller questions, and solve them one by one. For example, the original question is:\nhow many orders in last three month? Which month has highest?\nYou will spit into several smaller questions:\n1.Calculate total orders of last three month.\n2.Calculate monthly total order of last three month and calculate which months order is highest. You MUST use the available tools everytime to answer the question",
            "prompt": "${parameters.question}"
        }
    },
    "memory": {
        "type": "conversation_index"
    },
    "parameters": {
        "_llm_interface": "bedrock/converse/claude"
    },
    "tools": [
        {
            "type": "ListIndexTool"
        },
        {
            "type": "SearchIndexTool"
        },
        {
            "type": "IndexMappingTool"
        }
    ],
    "app_type": "os_chat"
}

// Run agent execute stream API
POST /_plugins/_ml/agents/lIDfPJoBgU_xdMdyRea7/_execute/stream
{
    "parameters": {
        "question": "How many indices are in my cluster?"
    }
}

// Sample response
data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"{","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\n\t\"steps\": [\"","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"Use ListIndexTool to","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" get information","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" about all","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" indices in","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" the cluster\"],","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\n\t\"result","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\": \"\"\n}","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"{","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\n\t","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\"steps\": [],","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\n\t\"result","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\": \"Base","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"d on my","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" analysis, there are ","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"12 indices in","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" the cluster","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":".\\n\\nAnal","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"ysis steps","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" performed:\\n1","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":". I used ListIndexT","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"ool to retrieve","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" information about all indices","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" in the OpenSearch cluster without","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" specifying any particular","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" index name","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":", which returned the complete","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" list.\\n\\","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"nThe ","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"12 indices include","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":":\\n-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" 8","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" system indices (with","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" '.' prefix):","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" .plugins-ml-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"model-group,","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" .plugins-ml","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"-memory-message","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":", .plugins-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"ml-memory-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"meta, .plugins","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"-ml-config","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":", .plugins-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"ml-model,","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" .plugins-ml","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"-agent, .","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"plugins-ml-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"task, .op","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"endistro_","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"security\\n-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" 3","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" log/operational","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" indices: top_","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"queries-2025","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":".10.30","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"-55871,","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" top_queries-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"2025.10","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":".31-55","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"872, security-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"auditlog-","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"2025.10","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":".31\\n","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"- 1 data","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" index: amazon_","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"products_image_","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"embedding\\n\\","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"nAll indices are","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" currently in ","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"'green' health","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":" status and '","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"open' status.\"","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"\n}","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"response","dataAsMap":{"content":"","is_last":false}}]}]}

data: {"inference_results":[{"output":[{"name":"memory_id","result":"gyKMO5oBT9DbLBdP0lCU"},{"name":"parent_interaction_id","result":"hCKMO5oBT9DbLBdP0lDC"},{"name":"executor_agent_memory_id","result":"hSKMO5oBT9DbLBdP31B7"},{"name":"executor_agent_parent_interaction_id","result":"hiKMO5oBT9DbLBdP31CX"},{"name":"response","dataAsMap":{"content":"","is_last":true}}]}]}

Related Issues

Resolves #[Issue number to be closed when this PR is merged]

Check List

New functionality includes testing.
New functionality has been documented.
API changes companion pull request created.
Commits are signed per the DCO using --signoff.
Public documentation issue/PR created.

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check here.

Signed-off-by: Nathalie Jonathan <[email protected]>

ylwu-amzn · 2025-11-01T23:49:52Z

PER agent will use planner to generate plan and use executor to execute each the plan. Do we stream out the plan and the executor agent response for each step?

nathaliellenaa · 2025-11-02T00:11:54Z

PER agent will use planner to generate plan and use executor to execute each the plan. Do we stream out the plan and the executor agent response for each step?

No, we only stream out the plan agent responses and the final LLM responses. For example the streamed output will look like this:

{
	"steps": ["Use ListIndexTool to get information about all indices in the cluster"],
	"result": ""
}
{
	"steps": [],
	"result": "Based on my analysis, there are 12 indices in the cluster.\n\nAnalysis steps performed:\n1. I used ListIndexTool to retrieve information about all indices in the OpenSearch cluster without specifying any particular index name, which returned the complete list.\n\nThe 12 indices include:\n- 8 system indices (with '.' prefix): .plugins-ml-model-group, .plugins-ml-memory-message, .plugins-ml-memory-meta, .plugins-ml-config, .plugins-ml-model, .plugins-ml-agent, .plugins-ml-task, .opendistro_security\n- 3 log/operational indices: top_queries-2025.10.30-55871, top_queries-2025.10.31-55872, security-auditlog-2025.10.31\n- 1 data index: amazon_products_image_embedding\n\nAll indices are currently in 'green' health status and 'open' status."
}

ylwu-amzn · 2025-11-02T00:18:05Z

PER agent will use planner to generate plan and use executor to execute each the plan. Do we stream out the plan and the executor agent response for each step?

No, we only stream out the plan agent responses and the final LLM responses. For example the streamed output will look like this:

{
	"steps": ["Use ListIndexTool to get information about all indices in the cluster"],
	"result": ""
}
{
	"steps": [],
	"result": "Based on my analysis, there are 12 indices in the cluster.\n\nAnalysis steps performed:\n1. I used ListIndexTool to retrieve information about all indices in the OpenSearch cluster without specifying any particular index name, which returned the complete list.\n\nThe 12 indices include:\n- 8 system indices (with '.' prefix): .plugins-ml-model-group, .plugins-ml-memory-message, .plugins-ml-memory-meta, .plugins-ml-config, .plugins-ml-model, .plugins-ml-agent, .plugins-ml-task, .opendistro_security\n- 3 log/operational indices: top_queries-2025.10.30-55871, top_queries-2025.10.31-55872, security-auditlog-2025.10.31\n- 1 data index: amazon_products_image_embedding\n\nAll indices are currently in 'green' health status and 'open' status."
}

So the streaming result contains each step's input and output, sounds good

ylwu-amzn · 2025-11-02T00:18:10Z

PER agent will use planner to generate plan and use executor to execute each the plan. Do we stream out the plan and the executor agent response for each step?

No, we only stream out the plan agent responses and the final LLM responses. For example the streamed output will look like this:

{
	"steps": ["Use ListIndexTool to get information about all indices in the cluster"],
	"result": ""
}
{
	"steps": [],
	"result": "Based on my analysis, there are 12 indices in the cluster.\n\nAnalysis steps performed:\n1. I used ListIndexTool to retrieve information about all indices in the OpenSearch cluster without specifying any particular index name, which returned the complete list.\n\nThe 12 indices include:\n- 8 system indices (with '.' prefix): .plugins-ml-model-group, .plugins-ml-memory-message, .plugins-ml-memory-meta, .plugins-ml-config, .plugins-ml-model, .plugins-ml-agent, .plugins-ml-task, .opendistro_security\n- 3 log/operational indices: top_queries-2025.10.30-55871, top_queries-2025.10.31-55872, security-auditlog-2025.10.31\n- 1 data index: amazon_products_image_embedding\n\nAll indices are currently in 'green' health status and 'open' status."
}

So the streaming result contains each step's input and output, sounds good

nathaliellenaa requested a deployment to ml-commons-cicd-env-require-approval November 1, 2025 00:49 — with GitHub Actions Waiting

nathaliellenaa added 2 commits November 1, 2025 10:35

Add PER streaming support

d8a9c75

Signed-off-by: Nathalie Jonathan <[email protected]>

Clean up, add tests

fd6a817

Signed-off-by: Nathalie Jonathan <[email protected]>

nathaliellenaa force-pushed the stream_per branch from 8080572 to fd6a817 Compare November 1, 2025 23:19

nathaliellenaa temporarily deployed to ml-commons-cicd-env-require-approval November 1, 2025 23:21 — with GitHub Actions Inactive

nathaliellenaa had a problem deploying to ml-commons-cicd-env-require-approval November 1, 2025 23:21 — with GitHub Actions Failure

nathaliellenaa had a problem deploying to ml-commons-cicd-env-require-approval November 1, 2025 23:21 — with GitHub Actions Error

nathaliellenaa marked this pull request as ready for review November 1, 2025 23:28

nathaliellenaa requested review from HenryL27, Zhangxunmt, austintlee, b4sjoo, dhrubo-os, jngz-es, mingshl, model-collapse, pyek-bot, rbhavna, sam-herman, xinyual, ylwu-amzn and zane-neo as code owners November 1, 2025 23:28

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Add streaming support for PER agent #4384

Add streaming support for PER agent #4384

nathaliellenaa commented Nov 1, 2025 •

edited

Loading

Uh oh!

ylwu-amzn commented Nov 1, 2025

Uh oh!

nathaliellenaa commented Nov 2, 2025

Uh oh!

ylwu-amzn commented Nov 2, 2025

Uh oh!

ylwu-amzn commented Nov 2, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

Add streaming support for PER agent #4384

Are you sure you want to change the base?

Add streaming support for PER agent #4384

Conversation

nathaliellenaa commented Nov 1, 2025 • edited Loading Uh oh! There was an error while loading. Please reload this page.

Uh oh!

Description

Related Issues

Check List

Uh oh!

ylwu-amzn commented Nov 1, 2025

Uh oh!

nathaliellenaa commented Nov 2, 2025

Uh oh!

ylwu-amzn commented Nov 2, 2025

Uh oh!

ylwu-amzn commented Nov 2, 2025

Uh oh!

Reviewers

Assignees

Labels

Projects

Milestone

Development

Uh oh!

2 participants

nathaliellenaa commented Nov 1, 2025 •

edited

Loading