faet: Displays real-time processing details (WIP)

Enhances user feedback during assistant message generation by displaying real-time processing status, including initialization, preparation, and token generation.

Provides detailed insights like context usage, temperature, and top-p settings, and speculative decoding status to the user.

Introduces a `useProcessingState` hook to manage and track the processing state. It utilizes a polling mechanism via slots service to get processing states.

Author: allozaur

tools/server/webui/src/lib/components/app/chat/ChatMessages/ChatMessage.svelte

@@ -7,6 +7,7 @@
 	import { copyToClipboard } from '$lib/utils/copy';
 	import { parseThinkingContent } from '$lib/utils/thinking';
 	import { isLoading } from '$lib/stores/chat.svelte';
+	import { useProcessingState } from '$lib/hooks/use-processing-state';
 	import { fade } from 'svelte/transition';
 
 	interface Props {
@@ -31,6 +32,8 @@
 	let editedContent = $state(message.content);
 	let textareaElement: HTMLTextAreaElement | undefined = $state();
 
+	const processingState = useProcessingState();
+
 	let thinkingContent = $derived.by(() => {
 		if (message.role === 'assistant') {
 			if (message.thinking) {
@@ -92,11 +95,19 @@
 	}
 
 	function handleSaveEdit() {
-		if (editedContent.trim() && editedContent !== message.content) {
+		if (editedContent.trim() !== message.content) {
 			onUpdateMessage?.(message, editedContent.trim());
 		}
 		isEditing = false;
 	}
+
+	$effect(() => {
+		if (message.role === 'assistant' && !message.content && isLoading()) {
+			processingState.startMonitoring();
+		} else {
+			processingState.stopMonitoring();
+		}
+	});
 </script>
 
 {#if message.role === 'user'}
@@ -186,8 +197,16 @@
 	{#if config?.role === 'assistant' && !message.content && isLoading()}
 		<div class="mx-auto w-full max-w-[48rem] mb-16" in:fade>
 			<span class="processing-text">
-				Processing
+				{processingState.getProcessingMessage()}
 			</span>
+			
+			{#if processingState.shouldShowDetails()}
+				<div class="processing-details">
+					{#each processingState.getProcessingDetails() as detail}
+						<span class="processing-detail">{detail}</span>
+					{/each}
+				</div>
+			{/if}
 		</div>
 	{/if}
 
@@ -260,6 +279,24 @@
 		-webkit-text-fill-color: transparent;
 		animation: shine 1s linear infinite;
 		font-weight: 500;
+		font-size: 0.875rem;
+	}
+
+	.processing-details {
+		display: flex;
+		flex-wrap: wrap;
+		justify-content: center;
+		gap: 0.75rem;
+		margin-top: 0.25rem;
+	}
+
+	.processing-detail {
+		color: var(--muted-foreground);
+		font-size: 0.75rem;
+		padding: 0.125rem 0.5rem;
+		background: var(--muted);
+		border-radius: 0.375rem;
+		font-family: ui-monospace, SFMono-Regular, "SF Mono", Consolas, "Liberation Mono", Menlo, monospace;
 	}
 
 	@keyframes shine {

tools/server/webui/src/lib/hooks/use-processing-state.ts

@@ -0,0 +1,95 @@
+import { slotsService } from '$lib/services/slots';
+import type { ApiProcessingState } from '$lib/types/api';
+
+export function useProcessingState() {
+	let processingState = $state<ApiProcessingState | null>(null);
+	let isPolling = $state(false);
+	let unsubscribe: (() => void) | null = null;
+
+	function startMonitoring(): void {
+		if (isPolling) return;
+
+		isPolling = true;
+		
+		unsubscribe = slotsService.subscribe((state) => {
+			processingState = state;
+		});
+
+		slotsService.startPolling();
+	}
+
+	function stopMonitoring(): void {
+		if (!isPolling) return;
+
+		isPolling = false;
+		processingState = null;
+
+		if (unsubscribe) {
+			unsubscribe();
+			unsubscribe = null;
+		}
+
+		slotsService.stopPolling();
+	}
+
+	function getProcessingMessage(): string {
+		if (!processingState) {
+			return 'Processing...';
+		}
+
+		switch (processingState.status) {
+			case 'initializing':
+				return 'Initializing...';
+			case 'preparing':
+				return 'Preparing response...';
+			case 'generating':
+				if (processingState.tokensDecoded > 0) {
+					return `Generating... (${processingState.tokensDecoded} tokens)`;
+				}
+				return 'Generating...';
+			default:
+				return 'Processing...';
+		}
+	}
+
+	function getProcessingDetails(): string[] {
+		if (!processingState) {
+			return [];
+		}
+
+		const details: string[] = [];
+
+		if (processingState.contextUsed > 0) {
+			const contextPercent = Math.round((processingState.contextUsed / processingState.contextTotal) * 100);
+			details.push(`Context: ${processingState.contextUsed}/${processingState.contextTotal} (${contextPercent}%)`);
+		}
+
+		if (processingState.temperature !== 0.8) {
+			details.push(`Temperature: ${processingState.temperature.toFixed(1)}`);
+		}
+		
+		if (processingState.topP !== 0.95) {
+			details.push(`Top-p: ${processingState.topP.toFixed(2)}`);
+		}
+
+		if (processingState.speculative) {
+			details.push('Speculative decoding enabled');
+		}
+
+		return details;
+	}
+
+	function shouldShowDetails(): boolean {
+		return processingState !== null && processingState.status !== 'idle';
+	}
+
+	return {
+		get processingState() { return processingState; },
+		get isPolling() { return isPolling; },
+		startMonitoring,
+		stopMonitoring,
+		getProcessingMessage,
+		getProcessingDetails,
+		shouldShowDetails
+	};
+}

tools/server/webui/src/lib/services/slots.ts

@@ -0,0 +1,122 @@
+import type { ApiSlotData, ApiProcessingState } from '$lib/types/api';
+
+export class SlotsService {
+	private baseUrl: string;
+	private pollingInterval: number;
+	private pollingTimer: number | null = null;
+	private callbacks: Set<(state: ApiProcessingState) => void> = new Set();
+
+	constructor(baseUrl = '', pollingInterval = 500) {
+		this.baseUrl = baseUrl;
+		this.pollingInterval = pollingInterval;
+	}
+
+	startPolling(): void {
+		if (this.pollingTimer) {
+			return;
+		}
+
+		this.poll();
+		this.pollingTimer = window.setInterval(() => {
+			this.poll();
+		}, this.pollingInterval);
+	}
+
+	stopPolling(): void {
+		if (this.pollingTimer) {
+			clearInterval(this.pollingTimer);
+			this.pollingTimer = null;
+		}
+	}
+
+	subscribe(callback: (state: ApiProcessingState) => void): () => void {
+		this.callbacks.add(callback);
+		return () => {
+			this.callbacks.delete(callback);
+		};
+	}
+
+	private async poll(): Promise<void> {
+		try {
+			const response = await fetch(`${this.baseUrl}/slots`);
+			if (!response.ok) {
+				console.warn('Failed to fetch slots data:', response.statusText);
+				return;
+			}
+
+			const slots: ApiSlotData[] = await response.json();
+			const processingState = this.parseProcessingState(slots);
+			
+			this.callbacks.forEach(callback => {
+				try {
+					callback(processingState);
+				} catch (error) {
+					console.error('Error in slots callback:', error);
+				}
+			});
+		} catch (error) {
+			console.warn('Error polling slots:', error);
+		}
+	}
+
+	private parseProcessingState(slots: ApiSlotData[]): ApiProcessingState {
+		const activeSlot = slots.find(slot => slot.id_task !== -1) || slots[0];
+
+		if (!activeSlot) {
+			return {
+				status: 'idle',
+				tokensDecoded: 0,
+				tokensRemaining: 0,
+				contextUsed: 0,
+				contextTotal: 4096,
+				temperature: 0.8,
+				topP: 0.95,
+				speculative: false,
+				hasNextToken: false
+			};
+		}
+
+		let status: ApiProcessingState['status'] = 'idle';
+		
+		if (activeSlot.is_processing) {
+			status = 'generating';
+		} else if (activeSlot.next_token.n_decoded === 0 && activeSlot.id_task !== -1) {
+			status = 'initializing';
+		} else if (!activeSlot.next_token.has_next_token && activeSlot.id_task !== -1) {
+			status = 'preparing';
+		}
+
+		// Calculate context usage (estimate based on prompt length and decoded tokens)
+		const promptTokens = Math.floor(activeSlot.prompt.length / 4); // Rough estimate
+		const contextUsed = promptTokens + activeSlot.next_token.n_decoded;
+
+		return {
+			status,
+			tokensDecoded: activeSlot.next_token.n_decoded,
+			tokensRemaining: activeSlot.next_token.n_remain,
+			contextUsed,
+			contextTotal: activeSlot.n_ctx,
+			temperature: activeSlot.params.temperature,
+			topP: activeSlot.params.top_p,
+			speculative: activeSlot.speculative,
+			hasNextToken: activeSlot.next_token.has_next_token
+		};
+	}
+
+	async getCurrentState(): Promise<ApiProcessingState | null> {
+		try {
+			const response = await fetch(`${this.baseUrl}/slots`);
+			if (!response.ok) {
+				return null;
+			}
+
+			const slots: ApiSlotData[] = await response.json();
+			return this.parseProcessingState(slots);
+		} catch (error) {
+			console.warn('Error fetching current slots state:', error);
+			return null;
+		}
+	}
+}
+
+export const slotsService = new SlotsService();

tools/server/webui/src/lib/types/api.d.ts

@@ -148,3 +148,81 @@ export interface ApiChatCompletionResponse {
 		};
 	}>;
 }
+
+export interface ApiSlotData {
+	id: number;
+	id_task: number;
+	n_ctx: number;
+	speculative: boolean;
+	is_processing: boolean;
+	params: {
+		n_predict: number;
+		seed: number;
+		temperature: number;
+		dynatemp_range: number;
+		dynatemp_exponent: number;
+		top_k: number;
+		top_p: number;
+		min_p: number;
+		top_n_sigma: number;
+		xtc_probability: number;
+		xtc_threshold: number;
+		typical_p: number;
+		repeat_last_n: number;
+		repeat_penalty: number;
+		presence_penalty: number;
+		frequency_penalty: number;
+		dry_multiplier: number;
+		dry_base: number;
+		dry_allowed_length: number;
+		dry_penalty_last_n: number;
+		dry_sequence_breakers: string[];
+		mirostat: number;
+		mirostat_tau: number;
+		mirostat_eta: number;
+		stop: string[];
+		max_tokens: number;
+		n_keep: number;
+		n_discard: number;
+		ignore_eos: boolean;
+		stream: boolean;
+		logit_bias: any[];
+		n_probs: number;
+		min_keep: number;
+		grammar: string;
+		grammar_lazy: boolean;
+		grammar_triggers: any[];
+		preserved_tokens: any[];
+		chat_format: string;
+		reasoning_format: string;
+		reasoning_in_content: boolean;
+		thinking_forced_open: boolean;
+		samplers: string[];
+		'speculative.n_max': number;
+		'speculative.n_min': number;
+		'speculative.p_min': number;
+		timings_per_token: boolean;
+		post_sampling_probs: boolean;
+		lora: any[];
+	};
+	prompt: string;
+	next_token: {
+		has_next_token: boolean;
+		has_new_line: boolean;
+		n_remain: number;
+		n_decoded: number;
+		stopping_word: string;
+	};
+}
+
+export interface ApiProcessingState {
+	status: 'initializing' | 'generating' | 'preparing' | 'idle';
+	tokensDecoded: number;
+	tokensRemaining: number;
+	contextUsed: number;
+	contextTotal: number;
+	temperature: number;
+	topP: number;
+	speculative: boolean;
+	hasNextToken: boolean;
+}