-
Notifications
You must be signed in to change notification settings - Fork 120
Expand file tree
/
Copy pathmetrics.go
More file actions
3771 lines (3358 loc) · 125 KB
/
Copy pathmetrics.go
File metadata and controls
3771 lines (3358 loc) · 125 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package madmin
import (
"cmp"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"maps"
"math"
"net/http"
"net/url"
"runtime/metrics"
"slices"
"sort"
"strconv"
"strings"
"time"
"github.com/shirou/gopsutil/v4/cpu"
"github.com/shirou/gopsutil/v4/load"
"github.com/tinylib/msgp/msgp"
)
//go:generate go tool msgp -unexported -d clearomitted -d "tag json" -d "timezone utc" -d "maps binkeys" -file $GOFILE
//msgp:replace HealItemType with:string
// MetricType is a bitfield representation of different metric types.
type MetricType uint32
// MetricsNone indicates no metrics.
const MetricsNone MetricType = 0
const (
MetricsScanner MetricType = 1 << iota
MetricsDisk
MetricsOS
MetricsBatchJobs
MetricsSiteResync
MetricNet
MetricsMem
MetricsCPU
MetricsRPC
MetricsRuntime
MetricsAPI
MetricsReplication
MetricsProcess
MetricsHealing
MetricsBuckets
MetricsKMS
MetricsTablesAPI
MetricsDistJobs
MetricsTargets
MetricsTier
MetricsILM
MetricsLocks
MetricsIAM
// MetricsAll must be last.
// Enables all metrics.
MetricsAll = 1<<iota - 1
)
// Contains returns whether m contains all of x.
func (m MetricType) Contains(x MetricType) bool {
return m&x == x
}
// String returns a comma separated list of flags as string.
func (m MetricType) String() string {
var b strings.Builder
addIf := func(cond bool, str string) {
if cond {
if b.Len() > 0 {
b.WriteByte(',')
}
b.WriteString(str)
}
}
addIf(m.Contains(MetricsScanner), "Scanner")
addIf(m.Contains(MetricsDisk), "Disk")
addIf(m.Contains(MetricsOS), "OS")
addIf(m.Contains(MetricsBatchJobs), "BatchJobs")
addIf(m.Contains(MetricsSiteResync), "SiteResync")
addIf(m.Contains(MetricNet), "Net")
addIf(m.Contains(MetricsMem), "Mem")
addIf(m.Contains(MetricsCPU), "CPU")
addIf(m.Contains(MetricsRPC), "RPC")
addIf(m.Contains(MetricsRuntime), "Runtime")
addIf(m.Contains(MetricsAPI), "API")
addIf(m.Contains(MetricsReplication), "Replication")
addIf(m.Contains(MetricsProcess), "Process")
addIf(m.Contains(MetricsHealing), "Healing")
addIf(m.Contains(MetricsBuckets), "Buckets")
addIf(m.Contains(MetricsKMS), "KMS")
addIf(m.Contains(MetricsTablesAPI), "Tables API")
addIf(m.Contains(MetricsDistJobs), "DistJobs")
addIf(m.Contains(MetricsTargets), "Targets")
addIf(m.Contains(MetricsTier), "Tier")
addIf(m.Contains(MetricsILM), "ILM")
addIf(m.Contains(MetricsLocks), "Locks")
addIf(m.Contains(MetricsIAM), "IAM")
return b.String()
}
// MetricFlags is a bitfield representation of different metric flags.
type MetricFlags uint64
const (
MetricsDayStats MetricFlags = 1 << iota // Include daily statistics (24h, 15-min segments)
MetricsByHost // Aggregate metrics by host/node.
MetricsByDisk // Aggregate metrics by disk.
MetricsLegacyDiskIO // Add legacy disk IO metrics.
MetricsByDiskSet // Aggregate metrics by disk pool+set index.
MetricsSMART // Include S.M.A.R.T. disk health data.
MetricsHourStats // Include last-hour statistics (1h, 1-min segments)
MetricsTopWarehouses // Include top-25 metrics by warehouse
MetricsTopNamespaces // Include top-25 metrics by namespace
MetricsTopTables // Include top-25 tables
MetricsTablesCatalog // Include the tables catalog inventory (leader-only; walks the catalog)
)
// Contains returns whether m contains all of x.
func (m MetricFlags) Contains(x MetricFlags) bool {
return m&x == x
}
// Add one or more flags to m.
func (m *MetricFlags) Add(x ...MetricFlags) {
for _, v := range x {
*m = *m | v
}
}
// String returns a comma separated list of flags as string.
func (m MetricFlags) String() string {
var b strings.Builder
addIf := func(cond bool, str string) {
if cond {
if b.Len() > 0 {
b.WriteByte(',')
}
b.WriteString(str)
}
}
addIf(m.Contains(MetricsDayStats), "DayStats")
addIf(m.Contains(MetricsByHost), "ByHost")
addIf(m.Contains(MetricsByDisk), "ByDisk")
addIf(m.Contains(MetricsLegacyDiskIO), "LegacyIO")
addIf(m.Contains(MetricsByDiskSet), "ByDiskSet")
addIf(m.Contains(MetricsSMART), "SMART")
addIf(m.Contains(MetricsHourStats), "HourStats")
addIf(m.Contains(MetricsTopWarehouses), "TopWarehouses")
addIf(m.Contains(MetricsTopNamespaces), "TopNamespaces")
addIf(m.Contains(MetricsTopTables), "TopTables")
addIf(m.Contains(MetricsTablesCatalog), "TablesCatalog")
return b.String()
}
// MetricsOptions are options provided to Metrics call.
type MetricsOptions struct {
Type MetricType // Return only these metric types. Several types can be combined using |. Leave at 0 to return all.
Flags MetricFlags // Flags to control returned metrics.
N int // Maximum number of samples to return. 0 will return endless stream.
Interval time.Duration // Interval between samples. Will be rounded up to 1s.
PoolIdx []int // Only include metrics for these pools. Leave empty for all.
Hosts []string // Only include specified hosts. Leave empty for all.
DrivePoolIdx []int // Only include metrics for these drive pools. Leave empty for all.
DriveSetIdx []int // Only include metrics for these drive sets (combine with PoolIdx if needed).
Disks []string // Include only specific disks. Leave empty for all.
Buckets []string // Include only specific buckets in bucket metrics. Leave empty for all.
ByJobID string
ByDepID string
// Alternative output merging.
// Populates maps of the same name in the result.
ByHost bool // Return individual metrics by host. Deprecated: use MetricsByHost instead.
ByDisk bool // Return individual metrics by disk. Deprecated: use MetricsByDisk instead.
}
// DriveSetPrefix will be used to select drives from specific sets.
const (
DriveSetPrefix = "::drive-set::"
DrivePoolPrefix = "::drive-pool::"
)
// Metrics makes an admin call to retrieve metrics.
// The provided function is called for each received entry.
func (adm *AdminClient) Metrics(ctx context.Context, o MetricsOptions, out func(RealtimeMetrics)) (err error) {
path := adminAPIPrefix + "/metrics"
q := make(url.Values)
q.Set("types", strconv.FormatUint(uint64(o.Type), 10))
q.Set("n", strconv.Itoa(o.N))
q.Set("interval", o.Interval.String())
q.Set("hosts", strings.Join(o.Hosts, ","))
if o.ByHost {
q.Set("by-host", "true") // Legacy flag
o.Flags.Add(MetricsByDisk)
}
for _, v := range o.DriveSetIdx {
o.Disks = append(o.Disks, fmt.Sprintf(DriveSetPrefix+"%d", v))
}
for _, v := range o.DrivePoolIdx {
o.Disks = append(o.Disks, fmt.Sprintf(DrivePoolPrefix+"%d", v))
}
q.Set("disks", strings.Join(o.Disks, ","))
if len(o.Buckets) > 0 {
q.Set("buckets", strings.Join(o.Buckets, ","))
}
if o.ByDisk {
q.Set("by-disk", "true") // Legacy flag
o.Flags.Add(MetricsByDisk)
}
if o.ByJobID != "" {
q.Set("by-jobID", o.ByJobID)
}
if o.ByDepID != "" {
q.Set("by-depID", o.ByDepID)
}
if len(o.PoolIdx) > 0 {
str := make([]string, len(o.PoolIdx))
for i, id := range o.PoolIdx {
str[i] = strconv.Itoa(id)
}
q.Set("pool-idx", strings.Join(str, ","))
}
q.Set("flags", strconv.FormatUint(uint64(o.Flags), 10))
resp, err := adm.executeMethod(ctx,
http.MethodGet, requestData{
customHeaders: map[string][]string{
"Accept": {"application/vnd.msgpack"},
},
relPath: path,
queryValues: q,
},
)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp)
}
defer closeResponse(resp)
// Choose decoder based on content type
var decodeOne func(m *RealtimeMetrics) error
switch resp.Header.Get("Content-Type") {
case "application/vnd.msgpack":
dec := msgp.NewReader(resp.Body)
decodeOne = func(m *RealtimeMetrics) error {
return m.DecodeMsg(dec)
}
default:
dec := json.NewDecoder(resp.Body)
decodeOne = func(m *RealtimeMetrics) error {
return dec.Decode(m)
}
}
for {
var m RealtimeMetrics
err := decodeOne(&m)
if err != nil {
if errors.Is(err, io.EOF) {
err = io.ErrUnexpectedEOF
}
return err
}
if m.CollectedAt.IsZero() {
m.CollectedAt = time.Now()
}
out(m)
if m.Final {
break
}
}
return nil
}
// RealtimeMetrics provides realtime metrics.
// This is intended to be expanded over time to cover more types.
type RealtimeMetrics struct {
// CollectedAt is the time these metrics were collected.
CollectedAt time.Time `json:"collected"`
// Error indicates an error occurred.
Errors []string `json:"errors,omitempty"`
// Hosts indicates the scanned hosts
Hosts []string `json:"hosts"`
// Aggregated contains aggregated metrics for all hosts
Aggregated Metrics `json:"aggregated"`
// ByHost contains metrics for each host if requested.
ByHost map[string]Metrics `json:"by_host,omitempty"`
// ByDisk contains metrics for each disk if requested.
ByDisk map[string]DiskMetric `json:"by_disk,omitempty"`
// ByDiskSet contains disk metrics aggregated by pool+set index.
ByDiskSet map[int]map[int]DiskMetric `json:"by_disk_set,omitempty"`
// Final indicates whether this is the final packet and the receiver can exit.
Final bool `json:"final"`
}
// Merge functionality:
//
// Overall rules: a.Merge(b)
//
// 1. All metrics must be accumulated and must be independent of order of merges.
// 2. If a field is not set in the other, it is not modified.
// 3. If a field is set in both, the value is merged.
// 4. Only a may be mutated.
// 5. 'a' can be the zero value.
// Merge will merge other into r.
func (r *RealtimeMetrics) Merge(other *RealtimeMetrics) {
if other == nil {
return
}
if r.CollectedAt.Before(other.CollectedAt) {
r.CollectedAt = other.CollectedAt
}
if len(other.Errors) > 0 {
r.Errors = append(r.Errors, other.Errors...)
}
if r.ByHost == nil && len(other.ByHost) > 0 {
r.ByHost = make(map[string]Metrics, len(other.ByHost))
}
for host, metrics := range other.ByHost {
r.ByHost[host] = metrics
}
r.Hosts = append(r.Hosts, other.Hosts...)
r.Aggregated.Merge(&other.Aggregated)
sort.Strings(r.Hosts)
// Gather per disk metrics
if r.ByDisk == nil && len(other.ByDisk) > 0 {
r.ByDisk = make(map[string]DiskMetric, len(other.ByDisk))
}
for disk, metrics := range other.ByDisk {
r.ByDisk[disk] = metrics
}
if r.ByDiskSet == nil && len(other.ByDiskSet) > 0 {
r.ByDiskSet = make(map[int]map[int]DiskMetric, len(other.ByDisk))
}
for pIdx, pool := range other.ByDiskSet {
dstp := r.ByDiskSet[pIdx]
if dstp == nil {
dstp = make(map[int]DiskMetric, len(pool))
r.ByDiskSet[pIdx] = dstp
}
for sIdx, disks := range pool {
dsts := dstp[sIdx]
dsts.Merge(&disks)
dstp[sIdx] = dsts
}
}
}
// Metrics contains all metric types.
type Metrics struct {
Scanner *ScannerMetrics `json:"scanner,omitempty"`
Disk *DiskMetric `json:"disk,omitempty"`
OS *OSMetrics `json:"os,omitempty"`
BatchJobs *BatchJobMetrics `json:"batchJobs,omitempty"`
SiteResync *SiteResyncMetrics `json:"siteResync,omitempty"`
Net *NetMetrics `json:"net,omitempty"`
Mem *MemMetrics `json:"mem,omitempty"`
CPU *CPUMetrics `json:"cpu,omitempty"`
RPC *RPCMetrics `json:"rpc,omitempty"`
Go *RuntimeMetrics `json:"go,omitempty"`
API *APIMetrics `json:"api,omitempty"`
Replication *ReplicationMetrics `json:"replication,omitempty"`
Process *ProcessMetrics `json:"process,omitempty"`
Healing *HealingMetrics `json:"healing,omitempty"`
Buckets *BucketAPIMetrics `json:"buckets,omitempty"`
KMS *KMSRtMetrics `json:"kms,omitempty"`
TablesAPI *TableAPIMetrics `json:"tables_api,omitempty"`
DistJobs *DistJobMetrics `json:"dist_jobs,omitempty"`
Targets *DeliveryTargetMetrics `json:"targets,omitempty"`
Tier *WarmTierMetrics `json:"tier,omitempty"`
ILM *ILMMetrics `json:"ilm,omitempty"`
Locks *LockMetrics `json:"locks,omitempty"`
IAM *IAMMetrics `json:"iam,omitempty"`
}
// Merge other into r.
func (r *Metrics) Merge(other *Metrics) {
if other == nil {
return
}
if r.Scanner == nil && other.Scanner != nil {
r.Scanner = &ScannerMetrics{}
}
r.Scanner.Merge(other.Scanner)
if r.Disk == nil && other.Disk != nil {
r.Disk = &DiskMetric{}
}
r.Disk.Merge(other.Disk)
if r.OS == nil && other.OS != nil {
r.OS = &OSMetrics{}
}
r.OS.Merge(other.OS)
if r.BatchJobs == nil && other.BatchJobs != nil {
r.BatchJobs = &BatchJobMetrics{}
}
r.BatchJobs.Merge(other.BatchJobs)
if r.SiteResync == nil && other.SiteResync != nil {
r.SiteResync = &SiteResyncMetrics{}
}
r.SiteResync.Merge(other.SiteResync)
if r.Net == nil && other.Net != nil {
r.Net = &NetMetrics{}
}
r.Net.Merge(other.Net)
if r.RPC == nil && other.RPC != nil {
r.RPC = &RPCMetrics{}
}
r.RPC.Merge(other.RPC)
if r.Go == nil && other.Go != nil {
r.Go = &RuntimeMetrics{}
}
r.Go.Merge(other.Go)
if r.API == nil && other.API != nil {
r.API = &APIMetrics{}
}
r.API.Merge(other.API)
if r.Replication == nil && other.Replication != nil {
r.Replication = &ReplicationMetrics{}
}
r.Replication.Merge(other.Replication)
if r.Mem == nil && other.Mem != nil {
r.Mem = &MemMetrics{}
}
r.Mem.Merge(other.Mem)
if r.CPU == nil && other.CPU != nil {
r.CPU = &CPUMetrics{}
}
r.CPU.Merge(other.CPU)
if r.Process == nil && other.Process != nil {
r.Process = &ProcessMetrics{}
}
r.Process.Merge(other.Process)
if r.Healing == nil && other.Healing != nil {
r.Healing = &HealingMetrics{}
}
r.Healing.Merge(other.Healing)
if r.Buckets == nil && other.Buckets != nil {
r.Buckets = &BucketAPIMetrics{}
}
r.Buckets.Merge(other.Buckets)
if other.KMS != nil {
if r.KMS == nil {
r.KMS = &KMSRtMetrics{}
}
r.KMS.Merge(other.KMS)
}
if other.TablesAPI != nil {
if r.TablesAPI == nil {
r.TablesAPI = &TableAPIMetrics{}
}
r.TablesAPI.Merge(other.TablesAPI)
}
if other.DistJobs != nil {
if r.DistJobs == nil {
r.DistJobs = &DistJobMetrics{}
}
r.DistJobs.Merge(other.DistJobs)
}
if other.Targets != nil {
if r.Targets == nil {
r.Targets = &DeliveryTargetMetrics{}
}
r.Targets.Merge(other.Targets)
}
if other.Tier != nil {
if r.Tier == nil {
r.Tier = &WarmTierMetrics{}
}
r.Tier.Merge(other.Tier)
}
if other.ILM != nil {
if r.ILM == nil {
r.ILM = &ILMMetrics{}
}
r.ILM.Merge(other.ILM)
}
if other.Locks != nil {
if r.Locks == nil {
r.Locks = &LockMetrics{}
}
r.Locks.Merge(other.Locks)
}
if other.IAM != nil {
if r.IAM == nil {
r.IAM = &IAMMetrics{}
}
r.IAM.Merge(other.IAM)
}
}
// BucketILMStats reports the cumulative ILM action counters for a single
// bucket carried in a ScannerMetrics value.
type BucketILMStats struct {
Bucket string `json:"bucket,omitempty" msg:"bucket,omitempty"`
ActionCounters map[string]uint64 `json:"action_counters,omitempty" msg:"action_counters,omitempty"`
}
// Merge adds other.ActionCounters into b.ActionCounters. b.Bucket is preserved
// when set, otherwise adopted from other (even when other has no counters). A
// nil other is a no-op.
func (b *BucketILMStats) Merge(other *BucketILMStats) {
if other == nil {
return
}
if b.Bucket == "" {
b.Bucket = other.Bucket
}
if len(other.ActionCounters) == 0 {
return
}
if b.ActionCounters == nil {
b.ActionCounters = make(map[string]uint64, len(other.ActionCounters))
}
for action, n := range other.ActionCounters {
b.ActionCounters[action] += n
}
}
// ScannerMetrics contains scanner information.
type ScannerMetrics struct {
// Time these metrics were collected
CollectedAt time.Time `json:"collected"`
// Number of buckets currently scanning
OngoingBuckets int `json:"ongoing_buckets"`
// Stats per bucket, a map between bucket name and scan stats in all erasure sets
PerBucketStats map[string][]BucketScanInfo `json:"per_bucket_stats,omitempty"`
// Number of accumulated operations by type since server restart.
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
// Number of accumulated ILM operations by type since server restart.
LifeTimeILM map[string]uint64 `json:"ilm_ops,omitempty"`
// BucketLifeTimeILM reports cumulative ILM action counters per bucket,
// keyed by bucket name. Populated only when a specific bucket is
// requested; nil otherwise.
BucketLifeTimeILM map[string]*BucketILMStats `json:"bucket_ilm_stats,omitempty"`
// Last minute operation statistics.
LastMinute struct {
// Scanner actions.
Actions map[string]TimedAction `json:"actions,omitempty"`
// ILM actions.
ILM map[string]TimedAction `json:"ilm,omitempty"`
} `json:"last_minute"`
// LastDay operation statistics.
LastDay map[string]SegmentedActions `json:"last_day,omitempty"`
// Currently active path(s) being scanned.
ActivePaths []string `json:"active,omitempty"`
// ExcessivePrefixes lists prefixes marked as having excessive sub-entries
// within the last 24 hours.
ExcessivePrefixes []string `json:"excessive,omitempty"`
// ExcessiveVersionObjects lists objects that have exceeded the version
// count or cumulative size threshold within the last 24 hours.
// Capped at 100 entries per cross-node merge; see DiscardedExcessEntries.
ExcessiveVersionObjects []string `json:"excessive_versions,omitempty"`
// DiscardedExcessEntries counts entries dropped beyond the 100-entry cap
// during cross-node merge. This counter is not deduplicated.
DiscardedExcessEntries uint64 `json:"discarded_excess_entries,omitempty"`
// Number of queued ILM expiry tasks.
ILMExpiryPendingTasks int `json:"ilm_expiry_pending_tasks,omitempty"`
// ILMExpiryTasksServiced tracks the last-minute latency and count of ILM expiry
// tasks that have been serviced, measured from queue time to completion.
ILMExpiryTasksServiced TimedAction `json:"ilm_expiry_tasks_cleanup"`
// QueuedForExpiry holds the most recently queued expiry objects
QueuedForExpiry []ExpiryObject `json:"queued_for_expiry,omitempty"`
}
// ExpiryObject contains information about an object recently queued for ILM expiry.
type ExpiryObject struct {
Bucket string `json:"bucket"`
Object string `json:"object"`
Versions int `json:"versions"`
QueuedAt time.Time `json:"queued_at"`
}
// Merge combines two lists of expiry objects into a single sorted list
// preserving order (newest first), the out is limited to max 25 objects.
func Merge(a, b []ExpiryObject) []ExpiryObject {
a = append(a, b...)
slices.SortFunc(a, func(a, b ExpiryObject) int {
res := b.QueuedAt.Compare(a.QueuedAt)
if res != 0 {
return res
}
res = cmp.Compare(a.Bucket, b.Bucket)
if res != 0 {
return res
}
return cmp.Compare(a.Object, b.Object)
})
return a[:min(len(a), 25)]
}
// SegmentedActions are time segmented scanner activity.
type SegmentedActions = Segmented[TimedAction, *TimedAction]
// Merge other into 's'.
func (s *ScannerMetrics) Merge(other *ScannerMetrics) {
if other == nil {
return
}
if s.CollectedAt.Before(other.CollectedAt) {
// Use latest timestamp
s.CollectedAt = other.CollectedAt
}
if s.OngoingBuckets < other.OngoingBuckets {
s.OngoingBuckets = other.OngoingBuckets
}
if s.PerBucketStats == nil {
s.PerBucketStats = make(map[string][]BucketScanInfo)
}
for bucket, otherSt := range other.PerBucketStats {
if len(otherSt) == 0 {
continue
}
_, ok := s.PerBucketStats[bucket]
if !ok {
s.PerBucketStats[bucket] = otherSt
}
}
// Regular ops
if len(other.LifeTimeOps) > 0 && s.LifeTimeOps == nil {
s.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
}
for k, v := range other.LifeTimeOps {
total := s.LifeTimeOps[k] + v
s.LifeTimeOps[k] = total
}
if s.LastMinute.Actions == nil && len(other.LastMinute.Actions) > 0 {
s.LastMinute.Actions = make(map[string]TimedAction, len(other.LastMinute.Actions))
}
for k, v := range other.LastMinute.Actions {
total := s.LastMinute.Actions[k]
total.Merge(v)
s.LastMinute.Actions[k] = total
}
if s.LastDay == nil && len(other.LastDay) > 0 {
s.LastDay = make(map[string]SegmentedActions, len(other.LastDay))
}
for k, v := range other.LastDay {
total := s.LastDay[k]
total.Add(&v)
s.LastDay[k] = total
}
// ILM
if len(other.LifeTimeILM) > 0 && s.LifeTimeILM == nil {
s.LifeTimeILM = make(map[string]uint64, len(other.LifeTimeILM))
}
for k, v := range other.LifeTimeILM {
total := s.LifeTimeILM[k] + v
s.LifeTimeILM[k] = total
}
for bucket, otherStats := range other.BucketLifeTimeILM {
if s.BucketLifeTimeILM == nil {
s.BucketLifeTimeILM = make(map[string]*BucketILMStats, len(other.BucketLifeTimeILM))
}
dst, ok := s.BucketLifeTimeILM[bucket]
if !ok {
dst = &BucketILMStats{Bucket: bucket}
s.BucketLifeTimeILM[bucket] = dst
}
dst.Merge(otherStats)
}
if s.LastMinute.ILM == nil && len(other.LastMinute.ILM) > 0 {
s.LastMinute.ILM = make(map[string]TimedAction, len(other.LastMinute.ILM))
}
for k, v := range other.LastMinute.ILM {
total := s.LastMinute.ILM[k]
total.Merge(v)
s.LastMinute.ILM[k] = total
}
s.ActivePaths = append(s.ActivePaths, other.ActivePaths...)
sort.Strings(s.ActivePaths)
if len(other.ExcessivePrefixes) > 0 {
merged := make(map[string]struct{}, len(s.ExcessivePrefixes)+len(other.ExcessivePrefixes))
for _, prefix := range s.ExcessivePrefixes {
merged[prefix] = struct{}{}
}
for _, prefix := range other.ExcessivePrefixes {
merged[prefix] = struct{}{}
}
s.ExcessivePrefixes = make([]string, 0, len(merged))
for prefix := range merged {
s.ExcessivePrefixes = append(s.ExcessivePrefixes, prefix)
}
sort.Strings(s.ExcessivePrefixes)
}
if len(other.ExcessiveVersionObjects) > 0 {
const maxExcessEntries = 100
seen := make(map[string]struct{}, len(s.ExcessiveVersionObjects)+len(other.ExcessiveVersionObjects))
for _, v := range s.ExcessiveVersionObjects {
seen[v] = struct{}{}
}
for _, v := range other.ExcessiveVersionObjects {
seen[v] = struct{}{}
}
keys := make([]string, 0, len(seen))
for k := range seen {
keys = append(keys, k)
}
sort.Strings(keys)
if len(keys) > maxExcessEntries {
s.DiscardedExcessEntries += uint64(len(keys) - maxExcessEntries)
keys = keys[:maxExcessEntries]
}
s.ExcessiveVersionObjects = keys
}
s.DiscardedExcessEntries += other.DiscardedExcessEntries
s.ILMExpiryPendingTasks += other.ILMExpiryPendingTasks
s.ILMExpiryTasksServiced.Merge(other.ILMExpiryTasksServiced)
if len(other.QueuedForExpiry) > 0 {
s.QueuedForExpiry = Merge(s.QueuedForExpiry, other.QueuedForExpiry)
}
}
// DiskIOStats contains IO stats of a single drive
type DiskIOStats struct {
N int `json:"n,omitempty"`
// WithIOStats is the subset of N whose kernel sysfs iostat
// (`/sys/dev/block/.../stat`) was readable; consumers gate I/O UI on it.
WithIOStats int `json:"with_iostats,omitempty"`
ReadIOs uint64 `json:"read_ios,omitempty"`
ReadMerges uint64 `json:"read_merges,omitempty"`
ReadSectors uint64 `json:"read_sectors,omitempty"`
ReadTicks uint64 `json:"read_ticks,omitempty"`
WriteIOs uint64 `json:"write_ios,omitempty"`
WriteMerges uint64 `json:"write_merges,omitempty"`
WriteSectors uint64 `json:"write_sectors,omitempty"`
WriteTicks uint64 `json:"write_ticks,omitempty"`
CurrentIOs uint64 `json:"current_ios,omitempty"`
TotalTicks uint64 `json:"total_ticks,omitempty"`
ReqTicks uint64 `json:"req_ticks,omitempty"`
DiscardIOs uint64 `json:"discard_ios,omitempty"`
DiscardMerges uint64 `json:"discard_merges,omitempty"`
DiscardSectors uint64 `json:"discard_sectors,omitempty"`
DiscardTicks uint64 `json:"discard_ticks,omitempty"`
FlushIOs uint64 `json:"flush_ios,omitempty"`
FlushTicks uint64 `json:"flush_ticks,omitempty"`
BitrotDetected uint64 `json:"bitrot_detected,omitempty"`
BitrotHealed uint64 `json:"bitrot_healed,omitempty"`
}
// DiskIOStatsLegacy mirrors DiskIOStats field-for-field so direct Go type
// conversions (used in DiskMetric.Merge) stay valid; mirror new fields too.
type DiskIOStatsLegacy struct {
N int `json:"n,omitempty"`
WithIOStats int `json:"with_iostats,omitempty"`
ReadIOs uint64 `json:"read_ios,omitempty"`
ReadMerges uint64 `json:"read_merges,omitempty"`
ReadSectors uint64 `json:"read_sectors,omitempty"`
ReadTicks uint64 `json:"read_ticks,omitempty"`
WriteIOs uint64 `json:"write_ios,omitempty"`
WriteMerges uint64 `json:"write_merges,omitempty"`
WriteSectors uint64 `json:"wrte_sectors,omitempty"` // note "spelling"
WriteTicks uint64 `json:"write_ticks,omitempty"`
CurrentIOs uint64 `json:"current_ios,omitempty"`
TotalTicks uint64 `json:"total_ticks,omitempty"`
ReqTicks uint64 `json:"req_ticks,omitempty"`
DiscardIOs uint64 `json:"discard_ios,omitempty"`
DiscardMerges uint64 `json:"discard_merges,omitempty"`
DiscardSectors uint64 `json:"discard_secotrs,omitempty"` // note "spelling"
DiscardTicks uint64 `json:"discard_ticks,omitempty"`
FlushIOs uint64 `json:"flush_ios,omitempty"`
FlushTicks uint64 `json:"flush_ticks,omitempty"`
BitrotDetected uint64 `json:"bitrot_detected,omitempty"`
BitrotHealed uint64 `json:"bitrot_healed,omitempty"`
}
// Add 'other' to 'd'.
func (d *DiskIOStats) Add(other *DiskIOStats) {
if other == nil || other.overflowed() {
// Discard segments carrying a uint64 underflow artifact (see overflowed)
// instead of polluting the aggregate.
return
}
if d.overflowed() {
// A corrupt receiver would otherwise drag its uint64 underflow artifact
// into every subsequent sum; reset it and restart the aggregate from the
// valid other value.
*d = DiskIOStats{}
}
d.N += other.N
d.WithIOStats += other.WithIOStats
d.ReadIOs += other.ReadIOs
d.ReadMerges += other.ReadMerges
d.ReadSectors += other.ReadSectors
d.ReadTicks += other.ReadTicks
d.WriteIOs += other.WriteIOs
d.WriteMerges += other.WriteMerges
d.WriteSectors += other.WriteSectors
d.WriteTicks += other.WriteTicks
d.CurrentIOs += other.CurrentIOs
d.TotalTicks += other.TotalTicks
d.ReqTicks += other.ReqTicks
d.DiscardIOs += other.DiscardIOs
d.DiscardMerges += other.DiscardMerges
d.DiscardSectors += other.DiscardSectors
d.DiscardTicks += other.DiscardTicks
d.FlushIOs += other.FlushIOs
d.FlushTicks += other.FlushTicks
d.BitrotDetected += other.BitrotDetected
d.BitrotHealed += other.BitrotHealed
}
// overflowed reports whether any counter has its high bit set (value >
// math.MaxInt64) — the signature of a uint64 underflow. Kernel iostat deltas
// underflow to ~MaxUint64 on counter resets (reboot, drive hot-swap, wrap),
// which is nonsensical as an IO count, so such segments are discarded on merge.
func (d *DiskIOStats) overflowed() bool {
return (d.ReadIOs | d.ReadMerges | d.ReadSectors | d.ReadTicks |
d.WriteIOs | d.WriteMerges | d.WriteSectors | d.WriteTicks |
d.CurrentIOs | d.TotalTicks | d.ReqTicks |
d.DiscardIOs | d.DiscardMerges | d.DiscardSectors | d.DiscardTicks |
d.FlushIOs | d.FlushTicks | d.BitrotDetected | d.BitrotHealed) > math.MaxInt64
}
// discardOverflowedSegments zeroes IO segments carrying a uint64 underflow
// artifact (see DiskIOStats.overflowed) so they do not survive a merge.
func discardOverflowedSegments(segs []DiskIOStats) {
for i := range segs {
if segs[i].overflowed() {
segs[i] = DiskIOStats{}
}
}
}
type (
SegmentedDiskActions = Segmented[DiskAction, *DiskAction]
SegmentedDiskIO = Segmented[DiskIOStats, *DiskIOStats]
)
// DiskMetric contains metrics for one or more disks.
type DiskMetric struct {
// Time these metrics were collected
CollectedAt time.Time `json:"collected"`
// Number of disks
NDisks int `json:"n_disks"`
// DiskIdx will be populated if all disks in the metrics have the same drive index.
DiskIdx *int `json:"disk_idx,omitempty"`
// SetIdx will be populated if all disks in the metrics are part of the same set.
SetIdx *int `json:"set_idx,omitempty"`
// PoolIdx will be populated if all disks in the metrics are part of the same pool.
PoolIdx *int `json:"pool_idx,omitempty"`
// Disk states for non-ok disks.
// See madmin.DriveState for possible values.
State map[string]int `json:"state,omitempty"`
// Offline disks
Offline int `json:"offline,omitempty"`
// Hanging - drives hanging.
Hanging int `json:"waiting,omitempty"`
// Healing disks
// Deprecated, will be removed in later releases
Healing int `json:"healing,omitempty"`
// HealingInfo gives us a high level overview of the drives healing state
HealingInfo *DriveHealInfo `json:"healingInfo,omitempty"`
// Cache stats if enabled.
Cache *CacheStats `json:"cache,omitempty"`
// Space info.
Space DriveSpaceInfo `json:"space"`
// Reclaim is background space reclamation on this drive: what the cleanup
// routines have deleted to give capacity back.
Reclaim DriveReclaimStats `json:"reclaim,omitempty"`
// Number of accumulated operations by type.
LifetimeOps map[string]DiskAction `json:"lifetime_ops,omitempty"`
// Last minute statistics.
LastMinute map[string]DiskAction `json:"last_minute,omitempty"`
// LastDaySegmented contains the segmented metrics for the last day.
LastDaySegmented map[string]SegmentedDiskActions `json:"last_day,omitempty"`
// LastHourSegmented contains the segmented metrics for the last hour.
LastHourSegmented map[string]SegmentedDiskActions `json:"last_hour,omitempty"`
// IO stats.
// Deprecated: use io_min, io_day instead.
IOStats *DiskIOStatsLegacy `json:"iostats,omitempty"`
// Rolling window last minute IO stats.
IOStatsMinute DiskIOStats `json:"io_min"`
// Rolling window daily IO stats (15-minute segments).
IOStatsDay SegmentedDiskIO `json:"io_day"`
// Rolling window hourly IO stats (1-minute segments).
IOStatsHour SegmentedDiskIO `json:"io_hour"`
// SMART health data for the disk.
SMART *SMARTInfo `json:"smart,omitempty"`
// Filesystem type (e.g. "xfs", "ext4" and count).
FSType map[string]int `json:"fsType,omitempty"`
}
type DriveHealInfo struct {
ItemsHealed uint64 `json:"itemsHealed"`
ItemsFailed uint64 `json:"itemsFailed"`
HealID string `json:"healID"`
Finished bool `json:"finished"`
Started time.Time `json:"started"`
Updated time.Time `json:"updated"`
}
// DriveSpaceInfo is the space info of one or more drives.
type DriveSpaceInfo struct {
N int `json:"n"`
Free TotalMinMaxUint64 `json:"free"`
Used TotalMinMaxUint64 `json:"used"`
UsedInodes TotalMinMaxUint64 `json:"used_inodes"`
FreeInodes TotalMinMaxUint64 `json:"free_inodes"`
}
func (d *DriveSpaceInfo) Merge(other DriveSpaceInfo) {
d.N += other.N
d.Free.Merge(other.Free, d.N)
d.Used.Merge(other.Used, d.N)
d.UsedInodes.Merge(other.UsedInodes, d.N)
d.FreeInodes.Merge(other.FreeInodes, d.N)
}
//msgp:tuple TotalMinMaxUint64
type TotalMinMaxUint64 struct {
Total uint64 `json:"total"`
Min uint64 `json:"min"`
Max uint64 `json:"max"`
}