Add histogram-based latency and TTFB reporting

cli/internal/results/summary/aggregate.go

@@ -97,8 +97,15 @@ type PhaseMetrics struct {
 	ThroughputLastOps float64
 	BandwidthAvgMBps  float64
 	BandwidthLastMBps float64
-	LatencyMeanMs     float64
+	LatencyHeadlineMs float64
 	LatencyMedianMs   float64
+	LatencyP90Ms      float64
+	LatencyP99Ms      float64
+	LatencyP999Ms     float64
+	TTFBMedianMs      float64
+	TTFBP90Ms         float64
+	TTFBP99Ms         float64
+	TTFBP999Ms        float64
 	ObjectSizeBytes   int64
 	ObjectSizeMB      float64
 	ObjectSizeGiB     float64
@@ -287,8 +294,15 @@ func deriveMetrics(stepData *StepData, objectSizeBytes int64) *PhaseMetrics {
 		ThroughputLastOps: row.ThroughputLastOps,
 		BandwidthAvgMBps:  row.BandwidthAvgMBps,
 		BandwidthLastMBps: row.BandwidthLastMBps,
-		LatencyMeanMs:     row.LatencyAvgMicros / 1000.0,
+		LatencyHeadlineMs: preferredLatencyMicros(row) / 1000.0,
 		LatencyMedianMs:   row.LatencyP50Micros / 1000.0,
+		LatencyP90Ms:      row.LatencyP90Micros / 1000.0,
+		LatencyP99Ms:      row.LatencyP99Micros / 1000.0,
+		LatencyP999Ms:     row.LatencyP999Micros / 1000.0,
+		TTFBMedianMs:      row.TTFBP50Micros / 1000.0,
+		TTFBP90Ms:         row.TTFBP90Micros / 1000.0,
+		TTFBP99Ms:         row.TTFBP99Micros / 1000.0,
+		TTFBP999Ms:        row.TTFBP999Micros / 1000.0,
 		Concurrency:       row.Concurrency,
 		ConcurrencyMean:   row.ConcurrencyMean,
 		NodeCount:         row.NodeCount,
@@ -304,6 +318,13 @@ func deriveMetrics(stepData *StepData, objectSizeBytes int64) *PhaseMetrics {
 	return metrics
 }
 
+func preferredLatencyMicros(row MetricsTotalsRow) float64 {
+	if row.LatencyP50Micros > 0 {
+		return row.LatencyP50Micros
+	}
+	return row.LatencyAvgMicros
+}
+
 func selectMetricsRow(totals *MetricsTotals, operationHint string) MetricsTotalsRow {
 	if totals == nil || len(totals.Rows) == 0 {
 		return MetricsTotalsRow{}

cli/internal/results/summary/metrics_totals.go

@@ -34,8 +34,19 @@ type MetricsTotalsRow struct {
 	BandwidthLastMBps   float64
 	DurationAvgMicros   float64
 	DurationP50Micros   float64
+	DurationP90Micros   float64
+	DurationP99Micros   float64
+	DurationP999Micros  float64
 	LatencyAvgMicros    float64
 	LatencyP50Micros    float64
+	LatencyP90Micros    float64
+	LatencyP99Micros    float64
+	LatencyP999Micros   float64
+	TTFBAvgMicros       float64
+	TTFBP50Micros       float64
+	TTFBP90Micros       float64
+	TTFBP99Micros       float64
+	TTFBP999Micros      float64
 	SampleTimestamp     string
 }
 
@@ -56,8 +67,19 @@ const (
 	columnBWLast          = "BWLast[MB/s]"
 	columnDurationAvg     = "DurationAvg[us]"
 	columnDurationP50     = "DurationQ_0.5[us]"
+	columnDurationP90     = "DurationQ_0.9[us]"
+	columnDurationP99     = "DurationQ_0.99[us]"
+	columnDurationP999    = "DurationQ_0.999[us]"
 	columnLatencyAvg      = "LatencyAvg[us]"
 	columnLatencyP50      = "LatencyQ_0.5[us]"
+	columnLatencyP90      = "LatencyQ_0.9[us]"
+	columnLatencyP99      = "LatencyQ_0.99[us]"
+	columnLatencyP999     = "LatencyQ_0.999[us]"
+	columnTTFBAvg         = "TtfbAvg[us]"
+	columnTTFBP50         = "TtfbQ_0.5[us]"
+	columnTTFBP90         = "TtfbQ_0.9[us]"
+	columnTTFBP99         = "TtfbQ_0.99[us]"
+	columnTTFBP999        = "TtfbQ_0.999[us]"
 )
 
 var requiredColumns = []string{
@@ -176,12 +198,23 @@ func parseMetricsTotalsRow(record []string, index map[string]int) (MetricsTotals
 	if row.DurationP50Micros, err = parseFloat(record, index, columnDurationP50); err != nil {
 		return row, err
 	}
+	row.DurationP90Micros, _ = parseFloat(record, index, columnDurationP90)
+	row.DurationP99Micros, _ = parseFloat(record, index, columnDurationP99)
+	row.DurationP999Micros, _ = parseFloat(record, index, columnDurationP999)
 	if row.LatencyAvgMicros, err = parseFloat(record, index, columnLatencyAvg); err != nil {
 		return row, err
 	}
 	if row.LatencyP50Micros, err = parseFloat(record, index, columnLatencyP50); err != nil {
 		return row, err
 	}
+	row.LatencyP90Micros, _ = parseFloat(record, index, columnLatencyP90)
+	row.LatencyP99Micros, _ = parseFloat(record, index, columnLatencyP99)
+	row.LatencyP999Micros, _ = parseFloat(record, index, columnLatencyP999)
+	row.TTFBAvgMicros, _ = parseFloat(record, index, columnTTFBAvg)
+	row.TTFBP50Micros, _ = parseFloat(record, index, columnTTFBP50)
+	row.TTFBP90Micros, _ = parseFloat(record, index, columnTTFBP90)
+	row.TTFBP99Micros, _ = parseFloat(record, index, columnTTFBP99)
+	row.TTFBP999Micros, _ = parseFloat(record, index, columnTTFBP999)
 
 	return row, nil
 }

cli/internal/results/summary/render.go

@@ -191,7 +191,7 @@ func (r *Renderer) renderPerformance(b *strings.Builder, summary *RunSummary) {
 }
 
 func (r *Renderer) performanceTable(summary *RunSummary) string {
-	headers := []string{"Phase", "Object Size", "Success", "Data Moved", "IOPS Avg", "Latency Mean", "Bandwidth Avg"}
+	headers := []string{"Phase", "Object Size", "Success", "Data Moved", "IOPS Avg", "Latency P50", "Bandwidth Avg"}
 	isList := strings.EqualFold(summary.Workload.Type, workloadTypeList)
 	if isList {
 		headers[4] = "Ops/s Avg"
@@ -212,7 +212,7 @@ func (r *Renderer) performanceTable(summary *RunSummary) string {
 			formatInt(m.SuccessCount),
 			formatBytesHuman(m.DataBytes),
 			formatNumber(m.ThroughputAvgOps, "ops/s"),
-			formatNumber(m.LatencyMeanMs, "ms"),
+			formatNumber(m.LatencyHeadlineMs, "ms"),
 			formatNumber(m.BandwidthAvgMBps, "MB/s"),
 		}
 		rows = append(rows, row)

cli/internal/results/summary/render_test.go

@@ -35,22 +35,22 @@ func TestRendererFullReportIncludesSections(t *testing.T) {
 			{
 				PhaseLabel: "Create",
 				Metrics: &PhaseMetrics{
-					SuccessCount:     2499,
-					DataBytes:        2620391424,
-					ThroughputAvgOps: 334.6964,
-					LatencyMeanMs:    79.9,
-					BandwidthAvgMBps: 334.6964,
-					ObjectSizeHuman:  "1.00 MiB",
+					SuccessCount:      2499,
+					DataBytes:         2620391424,
+					ThroughputAvgOps:  334.6964,
+					LatencyHeadlineMs: 79.9,
+					BandwidthAvgMBps:  334.6964,
+					ObjectSizeHuman:   "1.00 MiB",
 				},
 			},
 			{
 				PhaseLabel: "Delete",
 				Metrics: &PhaseMetrics{
-					SuccessCount:     2454,
-					DataBytes:        0,
-					ThroughputAvgOps: 1227,
-					LatencyMeanMs:    23.4,
-					BandwidthAvgMBps: 0,
+					SuccessCount:      2454,
+					DataBytes:         0,
+					ThroughputAvgOps:  1227,
+					LatencyHeadlineMs: 23.4,
+					BandwidthAvgMBps:  0,
 				},
 			},
 		},
@@ -115,12 +115,12 @@ func TestRendererCompactSnippetIncludesTable(t *testing.T) {
 			{
 				PhaseLabel: "Create",
 				Metrics: &PhaseMetrics{
-					ObjectSizeHuman:  "1.00 MiB",
-					SuccessCount:     2499,
-					DataBytes:        2620391424,
-					ThroughputAvgOps: 372,
-					LatencyMeanMs:    89.2,
-					BandwidthAvgMBps: 372,
+					ObjectSizeHuman:   "1.00 MiB",
+					SuccessCount:      2499,
+					DataBytes:         2620391424,
+					ThroughputAvgOps:  372,
+					LatencyHeadlineMs: 89.2,
+					BandwidthAvgMBps:  372,
 				},
 			},
 		},
@@ -175,10 +175,10 @@ func TestRendererListWorkloadDisplaysPrefix(t *testing.T) {
 			{
 				PhaseLabel: "List",
 				Metrics: &PhaseMetrics{
-					SuccessCount:     128,
-					ThroughputAvgOps: 256.4,
-					LatencyMeanMs:    12.5,
-					BandwidthAvgMBps: 0,
+					SuccessCount:      128,
+					ThroughputAvgOps:  256.4,
+					LatencyHeadlineMs: 12.5,
+					BandwidthAvgMBps:  0,
 				},
 			},
 		},

cli/tui/chart_generic.go

@@ -378,11 +378,16 @@ func ExtractOpsPerSec(m PerformanceMetric) int64 {
 	return m.OpsPerSec
 }
 
-// ExtractMeanLatency extracts the mean latency metric
-func ExtractMeanLatency(m PerformanceMetric) int64 {
+// ExtractDisplayLatency extracts the latency value selected for UI display.
+func ExtractDisplayLatency(m PerformanceMetric) int64 {
 	return m.MeanLatency
 }
 
+// ExtractMeanLatency is kept for older tests/helpers that still refer to the legacy name.
+func ExtractMeanLatency(m PerformanceMetric) int64 {
+	return ExtractDisplayLatency(m)
+}
+
 // ExtractFailureIncrement extracts the incremental failure count
 func ExtractFailureIncrement(m PerformanceMetric) int64 {
 	return m.FailureIncrement

cli/tui/chart_test.go

@@ -195,7 +195,7 @@ func TestGenericChartRenderer_NarrowWidth(t *testing.T) {
 }
 
 func TestLatencyChartRenderer(t *testing.T) {
-	cr := NewGenericChartRenderer("Mean Latency", "", ExtractMeanLatency, FormatLatencyValue)
+	cr := NewGenericChartRenderer("Display Latency", "", ExtractDisplayLatency, FormatLatencyValue)
 	mc := NewMetricsCollector()
 
 	// Add some test data with latency values

cli/tui/json_metrics_test.go

@@ -815,6 +815,30 @@ func TestJSONMetricsPerformanceMetricStructure(t *testing.T) {
 	}
 }
 
+func TestSchema3TimingPrefersLatencyP50(t *testing.T) {
+	step := newTestStep()
+	step.MetricsSchema = 3
+	step.Timing.LatencyMeanUs = 10_000
+	step.Timing.DurationMeanUs = 20_000
+	step.Timing.Latency = &JSONTimingStat{Count: 3, MeanUs: 10_000, P50Us: 2_500}
+	step.Timing.Duration = &JSONTimingStat{Count: 3, MeanUs: 20_000, P50Us: 3_500}
+
+	client := NewSptAPIClient("http://test")
+	metrics, err := client.ParseJSONMetrics(marshalSteps(t, []JSONMetricsStep{step}))
+	if err != nil {
+		t.Fatalf("ParseJSONMetrics failed: %v", err)
+	}
+	if len(metrics) != 1 {
+		t.Fatalf("expected 1 metric, got %d", len(metrics))
+	}
+	if metrics[0].MeanLatency != 2_500 {
+		t.Fatalf("expected schema-v3 latency display value to use p50, got %d", metrics[0].MeanLatency)
+	}
+	if metrics[0].MeanDuration != 3_500 {
+		t.Fatalf("expected schema-v3 duration display value to use p50, got %d", metrics[0].MeanDuration)
+	}
+}
+
 func TestParseJSONMetricsPrefersLatestSample(t *testing.T) {
 	older := newTestStep()
 	older.StepID = "create-step"

cli/tui/metrics.go

@@ -32,7 +32,7 @@ type PerformanceMetric struct {
 	StepTime           float64 // Step time in seconds
 	OpsPerSec          int64   // Last Rate [op/s] - primary chart metric
 	MBPerSec           int64   // Last Rate [MB/s] - secondary chart metric
-	MeanLatency        int64   // Mean Latency [us]
+	MeanLatency        int64   // Display latency [us]; schema 3 prefers p50 with mean fallback
 	MeanDuration       int64   // Mean Duration [us]
 	FailureIncrement   int64   // Incremental failures since last sample
 

cli/tui/model.go

@@ -173,7 +173,7 @@ func InitialModel() Model {
 		startTime:            time.Now(),
 		metricsCollector:     NewMetricsCollector(),
 		opsChartRenderer:     NewGenericChartRenderer("Operations/sec", "ops/s", ExtractOpsPerSec, FormatOpsValue),
-		latencyChartRenderer: NewGenericChartRenderer("Mean Latency", "", ExtractMeanLatency, FormatLatencyValue),
+		latencyChartRenderer: NewGenericChartRenderer("Latency P50", "", ExtractDisplayLatency, FormatLatencyValue),
 		failureChartRenderer: NewGenericChartRendererWithScale("Failure Rate", "failures", ExtractFailureIncrement, FormatFailureValue, PageScrollSize),
 		historicalIndex:      0, // Start in live mode
 		statusStyle:          lipgloss.NewStyle().Background(lipgloss.Color("238")).Foreground(lipgloss.Color("255")),

cli/tui/model_test.go

@@ -1057,8 +1057,8 @@ func TestChartIntegrationInTUI(t *testing.T) {
 	if !strings.Contains(view, "Operations/sec") {
 		t.Error("Expected view to contain Operations/sec chart title")
 	}
-	if !strings.Contains(view, "Mean Latency") {
-		t.Error("Expected view to contain Mean Latency chart title")
+	if !strings.Contains(view, "Latency P50") {
+		t.Error("Expected view to contain Latency P50 chart title")
 	}
 
 	// Should contain ops/s in the chart area

cli/tui/spt_api.go

@@ -114,8 +114,24 @@ type JSONMetricsBandwidth struct {
 
 // JSONMetricsTiming contains timing-related metrics
 type JSONMetricsTiming struct {
-	LatencyMeanUs  float64 `json:"latency_mean_us"`
-	DurationMeanUs float64 `json:"duration_mean_us"`
+	LatencyMeanUs  float64         `json:"latency_mean_us"`
+	DurationMeanUs float64         `json:"duration_mean_us"`
+	Latency        *JSONTimingStat `json:"latency,omitempty"`
+	Duration       *JSONTimingStat `json:"duration,omitempty"`
+	TTFB           *JSONTimingStat `json:"ttfb,omitempty"`
+}
+
+// JSONTimingStat contains schema 3 timing distribution fields.
+type JSONTimingStat struct {
+	Count         int64   `json:"count"`
+	MeanUs        float64 `json:"mean_us"`
+	MinUs         int64   `json:"min_us"`
+	P50Us         int64   `json:"p50_us"`
+	P90Us         int64   `json:"p90_us"`
+	P99Us         int64   `json:"p99_us"`
+	P999Us        int64   `json:"p999_us"`
+	MaxUs         int64   `json:"max_us"`
+	OverflowCount int64   `json:"overflow_count"`
 }
 
 // JSONMetricsConcurrency contains concurrency-related metrics
@@ -532,6 +548,14 @@ func (c *SptAPIClient) ParseJSONMetrics(data string) ([]*PerformanceMetric, erro
 
 // stepToMetric converts a validated JSONMetricsStep into a PerformanceMetric.
 func stepToMetric(step *JSONMetricsStep, scope string, sampleTimestamp time.Time) *PerformanceMetric {
+	latencyUs := step.Timing.LatencyMeanUs
+	if step.MetricsSchema >= 3 && step.Timing.Latency != nil && step.Timing.Latency.P50Us > 0 {
+		latencyUs = float64(step.Timing.Latency.P50Us)
+	}
+	durationUs := step.Timing.DurationMeanUs
+	if step.MetricsSchema >= 3 && step.Timing.Duration != nil && step.Timing.Duration.P50Us > 0 {
+		durationUs = float64(step.Timing.Duration.P50Us)
+	}
 	metric := &PerformanceMetric{
 		Timestamp:                time.UnixMilli(step.Timestamp),
 		SampleTimestamp:          sampleTimestamp,
@@ -546,8 +570,8 @@ func stepToMetric(step *JSONMetricsStep, scope string, sampleTimestamp time.Time
 		StepTime:                 step.ElapsedTimeSeconds,
 		OpsPerSec:                int64(math.Round(step.Operations.SuccessRateLast)),
 		MBPerSec:                 int64(step.Bandwidth.BytesRateLast / 1_000_000), // Convert bytes to MB
-		MeanLatency:              int64(math.Round(step.Timing.LatencyMeanUs)),    // Already in microseconds
-		MeanDuration:             int64(math.Round(step.Timing.DurationMeanUs)),   // Already in microseconds
+		MeanLatency:              int64(math.Round(latencyUs)),                    // Schema 3 prefers p50.
+		MeanDuration:             int64(math.Round(durationUs)),
 		CompletionPercent:        float64(step.CompletionPercent),
 		OverallCompletionPercent: float64(step.OverallCompletion),
 		Unbounded:                step.Unbounded,

engine/core/spt-base/build.gradle

@@ -62,6 +62,7 @@ dependencies {
 	implementation(libs.fastutil) {
 		transitive = false
 	}
+	implementation libs.hdr.histogram
 	annotationProcessor libs.log4j.core
 	implementation(
 		libs.commons.codec,

engine/core/spt-base/doc/design/architecture/README.md

@@ -127,12 +127,11 @@ We read it by ~16Mb chunks (if there is enough data) and synchronously put it in
 node. But as we aggregate data in parallel from workers, each chunk can be a bit less than 16Mb so that chunk finishes 
 at the end of line, so we don't mix lines from different workers.
 
-Since version 4.3.0 some of the metrics are also only aggregated after the test step. To avoid a continuous stream
-of metrics passed over to entry node the timing metrics (latency and duration) are only aggregated after the test step.
-While the test is running only the mean value is calculated and passed over. After the step is done remote temporary
-files are aggregated into local entry node temporary files. Then they are deleted on the remote workers. 
-After that on the entry node the files are first sorted and then merged. Spt takes the specified quantile values from the sorted values and then 
-deletes files on the entry node.
+Timing metrics are maintained as bounded in-memory histograms on each node. Duration, latency, and time-to-first-byte
+percentiles can therefore be reported while a step is running and in final summaries without reading and sorting raw
+per-operation timing files. In distributed mode the timing snapshot sent to the entry node includes compact compressed
+histogram payloads, and the entry node merges the histograms before calculating percentiles. Raw timing files are now a
+diagnostic artifact controlled by `--output-metrics-timing-persist`; normal percentile reporting does not depend on them.
 
 # 3. Concurrency