Skip to content

Commit bebf769

Browse files
committed
Add OpenTelemetry Processor and Exporter
It can replace bespoke tracing solution while keeping the current behavior and log format. Signed-off-by: Aleksander Mistewicz <[email protected]>
1 parent d3f136a commit bebf769

File tree

8 files changed

+462
-40
lines changed

8 files changed

+462
-40
lines changed

etcdctl/go.sum

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,10 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS
8888
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
8989
go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
9090
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
91-
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
92-
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
93-
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
94-
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
91+
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
92+
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
93+
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
94+
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
9595
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
9696
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
9797
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=

pkg/go.mod

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ require (
1111
github.com/spf13/pflag v1.0.10
1212
github.com/stretchr/testify v1.11.1
1313
go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0
14+
go.opentelemetry.io/otel v1.38.0
15+
go.opentelemetry.io/otel/sdk v1.38.0
1416
go.opentelemetry.io/otel/trace v1.38.0
1517
go.uber.org/zap v1.27.0
1618
golang.org/x/sys v0.36.0
@@ -20,9 +22,13 @@ require (
2022
require (
2123
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
2224
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
25+
github.com/go-logr/logr v1.4.3 // indirect
26+
github.com/go-logr/stdr v1.2.2 // indirect
27+
github.com/google/uuid v1.6.0 // indirect
2328
github.com/inconshreveable/mousetrap v1.1.0 // indirect
2429
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
25-
go.opentelemetry.io/otel v1.38.0 // indirect
30+
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
31+
go.opentelemetry.io/otel/metric v1.38.0 // indirect
2632
go.uber.org/multierr v1.11.0 // indirect
2733
golang.org/x/net v0.44.0 // indirect
2834
golang.org/x/text v0.29.0 // indirect

pkg/go.sum

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
77
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
88
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
99
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
10+
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
1011
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
1112
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
1213
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -41,10 +42,10 @@ go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
4142
go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
4243
go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
4344
go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
44-
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
45-
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
46-
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
47-
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
45+
go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
46+
go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
47+
go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
48+
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
4849
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
4950
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
5051
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

pkg/traceutil/exporter.go

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
// Copyright 2025 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package traceutil
16+
17+
import (
18+
"context"
19+
"fmt"
20+
"slices"
21+
"strings"
22+
23+
"go.opentelemetry.io/otel/attribute"
24+
"go.opentelemetry.io/otel/sdk/trace"
25+
"go.uber.org/zap"
26+
)
27+
28+
// LogExporter writes Span to specified Logger.
29+
type LogExporter struct {
30+
// Log is usually zap.Logger.Info.
31+
Log func(msg string, fields ...zap.Field)
32+
}
33+
34+
var _ trace.SpanExporter = (*LogExporter)(nil)
35+
36+
// NewLogExporter creates a new LogExporter which will write Spans as Log messages.
37+
func NewLogExporter(logger *zap.Logger) *LogExporter {
38+
if logger == nil {
39+
logger = zap.NewNop()
40+
}
41+
return &LogExporter{Log: logger.Info}
42+
}
43+
44+
func (e *LogExporter) ExportSpans(ctx context.Context, spans []trace.ReadOnlySpan) error {
45+
for _, span := range spans {
46+
msg, fields := logSpan(span)
47+
e.Log(msg, fields...)
48+
}
49+
return nil
50+
}
51+
52+
func (e *LogExporter) Shutdown(ctx context.Context) error {
53+
return nil
54+
}
55+
56+
func logSpan(s trace.ReadOnlySpan) (string, []zap.Field) {
57+
start := s.StartTime()
58+
end := s.EndTime()
59+
duration := end.Sub(start)
60+
events := s.Events()
61+
steps := make([]string, 0, len(events))
62+
slices.SortFunc(events, func(a, b trace.Event) int {
63+
return a.Time.Compare(b.Time)
64+
})
65+
for _, event := range events {
66+
step := fmt.Sprintf("%s %s [+%dms]",
67+
event.Name, writeAttrs(event.Attributes), event.Time.Sub(start).Milliseconds())
68+
steps = append(steps, step)
69+
}
70+
msg := fmt.Sprintf("trace[%s] %s", s.SpanContext().SpanID().String(), s.Name())
71+
72+
return msg, []zap.Field{
73+
zap.String("detail", writeAttrs(s.Attributes())),
74+
zap.Duration("duration", duration),
75+
zap.Time("start", s.StartTime()),
76+
zap.Time("end", s.EndTime()),
77+
zap.Strings("steps", steps),
78+
zap.Int("step_count", len(steps)),
79+
}
80+
}
81+
82+
func writeAttrs(attrs []attribute.KeyValue) string {
83+
if len(attrs) == 0 {
84+
return ""
85+
}
86+
var buf strings.Builder
87+
buf.WriteString("{")
88+
for _, attr := range attrs {
89+
buf.WriteString(string(attr.Key))
90+
buf.WriteString(":")
91+
buf.WriteString(attr.Value.Emit())
92+
buf.WriteString("; ")
93+
}
94+
buf.WriteString("}")
95+
return buf.String()
96+
}

pkg/traceutil/exporter_test.go

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// Copyright 2025 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package traceutil_test
16+
17+
import (
18+
"testing"
19+
"time"
20+
21+
"github.com/stretchr/testify/assert"
22+
"go.opentelemetry.io/otel/attribute"
23+
"go.opentelemetry.io/otel/sdk/trace"
24+
"go.opentelemetry.io/otel/sdk/trace/tracetest"
25+
"go.uber.org/zap"
26+
27+
"go.etcd.io/etcd/pkg/v3/traceutil"
28+
)
29+
30+
func TestLogSpan(t *testing.T) {
31+
duration := 123 * time.Second
32+
33+
startTime, _ := time.Parse("2006-01-02 15:04:05", "2025-01-01 00:00:00")
34+
endTime := startTime.Add(duration)
35+
36+
tests := []struct {
37+
span trace.ReadOnlySpan
38+
wantMsg string
39+
wantFields []zap.Field
40+
}{
41+
{
42+
span: tracetest.SpanStub{
43+
Name: "span_with_two_events",
44+
StartTime: startTime,
45+
EndTime: endTime,
46+
Attributes: []attribute.KeyValue{
47+
attribute.String("key1", "value1"),
48+
attribute.String("key2", "value2"),
49+
},
50+
Events: []trace.Event{
51+
{
52+
Time: startTime.Add(1 * time.Second),
53+
Name: "event1",
54+
Attributes: []attribute.KeyValue{attribute.String("key3", "value3")},
55+
},
56+
{
57+
Time: startTime.Add(2 * time.Second),
58+
Name: "event2",
59+
Attributes: []attribute.KeyValue{attribute.String("key4", "value4")},
60+
},
61+
},
62+
}.Snapshot(),
63+
wantMsg: "trace[0000000000000000] span_with_two_events",
64+
wantFields: []zap.Field{
65+
zap.String("detail", "{key1:value1; key2:value2; }"),
66+
zap.Duration("duration", duration),
67+
zap.Time("start", startTime),
68+
zap.Time("end", endTime),
69+
zap.Strings("steps", []string{"event1 {key3:value3; } [+1000ms]", "event2 {key4:value4; } [+2000ms]"}),
70+
zap.Int("step_count", 2),
71+
},
72+
},
73+
}
74+
75+
for _, tt := range tests {
76+
t.Run(tt.span.Name(), func(t *testing.T) {
77+
exporter := traceutil.LogExporter{
78+
Log: func(msg string, fields ...zap.Field) {
79+
assert.Equal(t, tt.wantMsg, msg)
80+
assert.Equal(t, tt.wantFields, fields)
81+
},
82+
}
83+
exporter.ExportSpans(t.Context(), []trace.ReadOnlySpan{tt.span})
84+
})
85+
}
86+
}

pkg/traceutil/processor.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Copyright 2025 The etcd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package traceutil
16+
17+
import (
18+
"context"
19+
"time"
20+
21+
"go.opentelemetry.io/otel/sdk/trace"
22+
)
23+
24+
// LongSpanProcessor is a SpanProcessor that passes to SpanExporter only Spans
25+
// with a duration longer than Threshold and operation in Allowlist.
26+
type LongSpanProcessor struct {
27+
trace.SpanExporter
28+
// Threshold is the duration under which spans are not logged.
29+
Threshold time.Duration
30+
// Allowlist specifies operations for which a log may be emitted.
31+
Allowlist map[string]bool
32+
}
33+
34+
var _ trace.SpanProcessor = (*LongSpanProcessor)(nil)
35+
36+
// NewLongSpanProcessor creates a new LongSpanProcessor which will pass to
37+
// SpanExporter all Spans with duration longer than Threshold and operation in
38+
// Allowlist.
39+
func NewLongSpanProcessor(exporter trace.SpanExporter, threshold time.Duration) *LongSpanProcessor {
40+
return &LongSpanProcessor{
41+
SpanExporter: exporter,
42+
Threshold: threshold,
43+
Allowlist: map[string]bool{
44+
"txn": true,
45+
"range": true,
46+
"put": true,
47+
"delete_range": true,
48+
"compact": true,
49+
"lease_grant": true,
50+
"lease_revoke": true,
51+
},
52+
}
53+
}
54+
55+
func (f LongSpanProcessor) OnStart(parent context.Context, s trace.ReadWriteSpan) {}
56+
func (f LongSpanProcessor) ForceFlush(ctx context.Context) error { return nil }
57+
func (f LongSpanProcessor) OnEnd(s trace.ReadOnlySpan) {
58+
if f.Threshold > 0 && s.EndTime().Sub(s.StartTime()) < f.Threshold {
59+
return
60+
}
61+
if f.Allowlist != nil && !f.Allowlist[s.Name()] {
62+
return
63+
}
64+
f.ExportSpans(context.Background(), []trace.ReadOnlySpan{s})
65+
}

0 commit comments

Comments
 (0)