Memory leak due to alive processes after calls hangup

[amateope]

Describe the bug

When making a SIP call, once the call is hung up, a participant disconnected event is received. At that point, the session is expected to terminate and all associated processes should be shut down and cleaned up.
However, the process does not exit as expected. Instead, it remains alive after the call has ended and continues consuming memory, leading to resource leakage over time.

Relevant log output

{"level":30,"time":1766150585732,"pid":99129,"hostname":"HBZF5B3","speech_id":"speech_05e26044-73b","message":"Yes, I can hear you loud and clear How can I help you today","msg":"playout completed without interruption"}
{"level":30,"time":1766150588006,"pid":99129,"hostname":"HBZF5B3","participant":"sip_1003","reason":"CLIENT_INITIATED","msg":"closing agent session due to participant disconnect (disable via `RoomInputOptions.closeOnDisconnect=False`)"}
{"level":30,"time":1766150588007,"pid":99129,"hostname":"HBZF5B3","msg":"mainTask: draining and no more speech tasks"}
{"level":30,"time":1766150588007,"pid":99129,"hostname":"HBZF5B3","msg":"AgentActivity mainTask: exiting"}
{"level":20,"time":1766150588009,"pid":99129,"hostname":"HBZF5B3","msg":"VAD task closed"}
{"level":30,"time":1766150588010,"pid":99129,"hostname":"HBZF5B3","reason":"participant_disconnected","error":null,"msg":"AgentSession closed"}
{"level":20,"time":1766150607677,"pid":99129,"hostname":"HBZF5B3","pid":99129,"jobID":"AJ_X6B7ZPfFxovJ","msg":"shutting down"}
{"level":20,"time":1766150607677,"pid":99068,"hostname":"HBZF5B3","msg":"job exiting"}
{"level":20,"time":1766150607677,"pid":99129,"hostname":"HBZF5B3","info":{"acceptArguments":{"name":"","identity":"agent-AJ_X6B7ZPfFxovJ","metadata":""},"job":{"id":"AJ_X6B7ZPfFxovJ","type":"JT_ROOM","room":{"sid":"RM_NchdUUtSKd9x","name":"1003_KxZrnqX69VBP","emptyTimeout":300,"maxParticipants":0,"creationTime":"1766150569","turnPassword":"password","enabledCodecs":[{"mime":"audio/PCMU","fmtpLine":""},{"mime":"audio/PCMA","fmtpLine":""},{"mime":"audio/opus","fmtpLine":""},{"mime":"audio/red","fmtpLine":""},{"mime":"video/VP8","fmtpLine":""},{"mime":"video/H264","fmtpLine":""},{"mime":"video/VP9","fmtpLine":""},{"mime":"video/AV1","fmtpLine":""},{"mime":"video/H265","fmtpLine":""},{"mime":"video/rtx","fmtpLine":""}],"metadata":"","numParticipants":0,"activeRecording":false,"numPublishers":0,"departureTimeout":20,"creationTimeMs":"1766150569406"},"namespace":"","metadata":"{}","agentName":"agent-name","state":{"status":"JS_RUNNING","error":"","startedAt":"1766150569608048142","endedAt":"0","updatedAt":"1766150569608048142","participantIdentity":"","workerId":"AW_goUgsY2ZBXnP","agentId":""},"dispatchId":"AD_KJCRG6WKFENk","enableRecording":false},"url":"ws://host:port","token":"ey...","workerId":"AW_goUgsY2ZBXnP"},"jobId":"AJ_X6B7ZPfFxovJ","roomId":"RM_NchdUUtSKd9x","eventsCount":17,"msg":"Session ended, report generated"}
{"level":40,"time":1766150666590,"pid":99068,"hostname":"HBZF5B3","msg":"job is unresponsive"}
{"level":20,"time":1766150666591,"pid":99129,"hostname":"HBZF5B3","pid":99129,"jobID":"AJ_X6B7ZPfFxovJ","msg":"SIGTERM received in job proc"}

Describe your environment

System:

OS: Linux 5.15 Ubuntu 20.04.6 LTS (Focal Fossa)
CPU: (8) x64 11th Gen Intel(R) Core(TM) i7-1165G7 @ 2.80GHz
Memory: 4.56 GB / 15.50 GB
Container: Yes
Shell: 5.8 - /usr/bin/zsh

Binaries:

Node: 22.17.1 - /run/user/1000/fnm_multishells/99661_1766150404893/bin/node
npm: 10.9.2 - /run/user/1000/fnm_multishells/99661_1766150404893/bin/npm
pnpm: 10.13.1 - /run/user/1000/fnm_multishells/99661_1766150404893/bin/pnpm

npmPackages:

@livekit/agents: 1.0.25 => 1.0.25
@livekit/agents-plugin-deepgram: 1.0.25 => 1.0.25
@livekit/agents-plugin-elevenlabs: 1.0.25 => 1.0.25
@livekit/agents-plugin-livekit: 1.0.25 => 1.0.25
@livekit/agents-plugin-openai: 1.0.25 => 1.0.25
@livekit/agents-plugin-silero: 1.0.25 => 1.0.25

Livekit Environment:

livekit/livekit-server:v1.9.6
livekit/sip:v1.2.0

Minimal reproducible example

import { fileURLToPath } from "node:url";

import { cli, defineAgent, type JobContext, type JobProcess, metrics, ServerOptions, voice } from "@livekit/agents";
import * as deepgram from "@livekit/agents-plugin-deepgram";
import * as elevenlabs from "@livekit/agents-plugin-elevenlabs";
import * as livekit from "@livekit/agents-plugin-livekit";
import * as openai from "@livekit/agents-plugin-openai";
import * as silero from "@livekit/agents-plugin-silero";
import dotenv from "dotenv";

dotenv.config();

class Assistant extends voice.Agent {
    constructor() {
        super({
            instructions: `You are a helpful voice AI assistant. The user is interacting with you via voice, even if you perceive the conversation as text.
      You eagerly assist users with their questions by providing information from your extensive knowledge.
      Your responses are concise, to the point, and without any complex formatting or punctuation including emojis, asterisks, or other symbols.
      You are curious, friendly, and have a sense of humor.`,
        });
    }
}

export default defineAgent({
    prewarm: async (proc: JobProcess) => {
        proc.userData.vad = await silero.VAD.load();
    },
    entry: async (ctx: JobContext) => {
        const session = new voice.AgentSession({
            stt: new deepgram.STT({
                interimResults: true,
                language: process.env.LOCALE ?? "en-GB",
                apiKey: process.env.DEEPGRAM_API_KEY ?? "",
            }),
            llm: openai.LLM.withAzure({
                apiKey: process.env.AZURE_OPENAI_API_KEY ?? "",
                apiVersion: process.env.AZURE_OPENAI_API_VERSION ?? "",
                azureEndpoint: process.env.AZURE_OPENAI_ENDPOINT ?? "",
                azureDeployment: process.env.AZURE_OPENAI_DEPLOYMENT ?? "",
                model: "gpt-4o-mini",
            }),
            tts: new elevenlabs.TTS({
                encoding: "pcm_16000",
                modelID: "eleven_turbo_v2_5",
                streamingLatency: 3,
                enableSsmlParsing: false,
                languageCode: process.env.LOCALE?.split("-")[0] ?? "en",
                chunkLengthSchedule: [80, 120, 200, 260],
                voice: {
                    id: process.env.ELEVEN_VOICE_ID ?? "",
                    name: process.env.ELEVEN_VOICE_NAME ?? "",
                    category: process.env.ELEVEN_CATEGORY ?? "premade",
                    settings: {
                        stability: 0.71,
                        similarity_boost: 0.5,
                        style: 0.0,
                        use_speaker_boost: true,
                    },
                },
            }),
            turnDetection: new livekit.turnDetector.MultilingualModel(),
            vad: ctx.proc.userData.vad! as silero.VAD,
            voiceOptions: {
                preemptiveGeneration: true,
            },
        });

        const usageCollector = new metrics.UsageCollector();
        session.on(voice.AgentSessionEventTypes.MetricsCollected, ev => {
            metrics.logMetrics(ev.metrics);
            usageCollector.collect(ev.metrics);
        });

        const logUsage = async () => {
            const summary = usageCollector.getSummary();
            console.log(`Usage: ${JSON.stringify(summary)}`);
        };

        ctx.addShutdownCallback(logUsage);

        await session.start({
            agent: new Assistant(),
            room: ctx.room,
            inputOptions: {},
        });
        await ctx.connect();
    },
});

cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url), agentName: process.env.AGENT_NAME }));

Additional information

After startup we have 4 node processes:

After hangup we have 5 node processes forever:

[amateope]

We have upgraded the dependencies to livekit 1.0.31 and the behaviour is better but it is not fully fixed. We see now that a hangup during the call is properly handled but if you hangup quickly at the beginning of the call, the application leaves a process running forever (same as version 1.0.25).

This is a screenshot of the startup (4 processes):

And this is a screenshot minutes after the hangup (5 processes):