Skip to content

Commit 1b877f4

Browse files
felixbarnyfelixbarny
authored
Add metrics OTLP endpoint skeleton (#133296)
1 parent 6448c1c commit 1b877f4

File tree

8 files changed

+452
-9
lines changed

8 files changed

+452
-9
lines changed

‎gradle/verification-metadata.xml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1751,6 +1751,16 @@
17511751
<sha256 value="2eaaac5f268b135f0e11dd30637d71df5751a0bb7ed6268659be57104d63122b" origin="Generated by Gradle"/>
17521752
</artifact>
17531753
</component>
1754+
<component group="io.opentelemetry" name="opentelemetry-api" version="1.53.0">
1755+
<artifact name="opentelemetry-api-1.53.0.jar">
1756+
<sha256 value="1991903b9fc76b27f1e6a70a5a97131668fb5f3ac9026178c450d510cbb1bef2" origin="Generated by Gradle"/>
1757+
</artifact>
1758+
</component>
1759+
<component group="io.opentelemetry" name="opentelemetry-common" version="1.53.0">
1760+
<artifact name="opentelemetry-common-1.53.0.jar">
1761+
<sha256 value="73a5181dd07e72c4312fadafc8328ad1046a32c050030e3a7b8c16113daad359" origin="Generated by Gradle"/>
1762+
</artifact>
1763+
</component>
17541764
<component group="io.opentelemetry" name="opentelemetry-context" version="1.31.0">
17551765
<artifact name="opentelemetry-context-1.31.0.jar">
17561766
<sha256 value="664896a5c34bcda20c95c8f45198a95e8f97a1cd5e5c2923978f42dddada787d" origin="Generated by Gradle"/>
@@ -1766,11 +1776,61 @@
17661776
<sha256 value="76f9dfe1a6f74d5081e07bde1f7cb9a06879d317ec0ae0f61dd8fb2be9afad4f" origin="Generated by Gradle"/>
17671777
</artifact>
17681778
</component>
1779+
<component group="io.opentelemetry" name="opentelemetry-context" version="1.53.0">
1780+
<artifact name="opentelemetry-context-1.53.0.jar">
1781+
<sha256 value="88a780c5bb7b51e7c07071d61403e34f13277d092bf441a5cb49cf2ce9c756e9" origin="Generated by Gradle"/>
1782+
</artifact>
1783+
</component>
1784+
<component group="io.opentelemetry" name="opentelemetry-exporter-common" version="1.53.0">
1785+
<artifact name="opentelemetry-exporter-common-1.53.0.jar">
1786+
<sha256 value="d7f093f987547c9c2c2caa28baf1254507c59b4dbf97b49fee5c9cd1ceb6f8d5" origin="Generated by Gradle"/>
1787+
</artifact>
1788+
</component>
1789+
<component group="io.opentelemetry" name="opentelemetry-exporter-otlp" version="1.53.0">
1790+
<artifact name="opentelemetry-exporter-otlp-1.53.0.jar">
1791+
<sha256 value="0ce88bf35577894ee185f718f1b981938bbfc8981f3b7fcecbfd7b7c375bc236" origin="Generated by Gradle"/>
1792+
</artifact>
1793+
</component>
1794+
<component group="io.opentelemetry" name="opentelemetry-exporter-otlp-common" version="1.53.0">
1795+
<artifact name="opentelemetry-exporter-otlp-common-1.53.0.jar">
1796+
<sha256 value="32bcf0e855110b09cffbb5aac59c69d2caaa35de646e07c0a5f7f50ab42d4c1f" origin="Generated by Gradle"/>
1797+
</artifact>
1798+
</component>
1799+
<component group="io.opentelemetry" name="opentelemetry-exporter-sender-jdk" version="1.53.0">
1800+
<artifact name="opentelemetry-exporter-sender-jdk-1.53.0.jar">
1801+
<sha256 value="d147fd99aa23be8b0ea8acbc06ffae2f091af2edfffeb44163ab79765f7e3b42" origin="Generated by Gradle"/>
1802+
</artifact>
1803+
</component>
17691804
<component group="io.opentelemetry" name="opentelemetry-sdk" version="1.47.0">
17701805
<artifact name="opentelemetry-sdk-1.47.0.jar">
17711806
<sha256 value="4a09eb2ee484769973e14218a34e6da54f35955aa02b26dc5238b0c2ed6a801d" origin="Generated by Gradle"/>
17721807
</artifact>
17731808
</component>
1809+
<component group="io.opentelemetry" name="opentelemetry-sdk" version="1.53.0">
1810+
<artifact name="opentelemetry-sdk-1.53.0.jar">
1811+
<sha256 value="d58721063bca5d612bf1f6dccb883664210999c5aa470c149d1509672f13229c" origin="Generated by Gradle"/>
1812+
</artifact>
1813+
</component>
1814+
<component group="io.opentelemetry" name="opentelemetry-sdk-common" version="1.53.0">
1815+
<artifact name="opentelemetry-sdk-common-1.53.0.jar">
1816+
<sha256 value="a5d68aca0920aa0dc17b9ecc5510abe4c344e66eae9a4420c992298e4712d3bc" origin="Generated by Gradle"/>
1817+
</artifact>
1818+
</component>
1819+
<component group="io.opentelemetry" name="opentelemetry-sdk-extension-autoconfigure" version="1.53.0">
1820+
<artifact name="opentelemetry-sdk-extension-autoconfigure-1.53.0.jar">
1821+
<sha256 value="72becaccfb3d79d91c6486bf4e68f2474a1aa7742934fb675ea077e35a8dc3e1" origin="Generated by Gradle"/>
1822+
</artifact>
1823+
</component>
1824+
<component group="io.opentelemetry" name="opentelemetry-sdk-extension-autoconfigure-spi" version="1.53.0">
1825+
<artifact name="opentelemetry-sdk-extension-autoconfigure-spi-1.53.0.jar">
1826+
<sha256 value="e814e63dc2f8cbdf84574b6289eb543b89b06486d03cad1e697b06d802ce27bb" origin="Generated by Gradle"/>
1827+
</artifact>
1828+
</component>
1829+
<component group="io.opentelemetry" name="opentelemetry-sdk-metrics" version="1.53.0">
1830+
<artifact name="opentelemetry-sdk-metrics-1.53.0.jar">
1831+
<sha256 value="3d7dbae6c03be035e7c4dbdd50442741b260be5463fa9aadca1b00af0efddee5" origin="Generated by Gradle"/>
1832+
</artifact>
1833+
</component>
17741834
<component group="io.opentelemetry" name="opentelemetry-semconv" version="1.21.0-alpha">
17751835
<artifact name="opentelemetry-semconv-1.21.0-alpha.jar">
17761836
<sha256 value="4a8f41b93eec51e85fa6b48e43de6785b742316fdd9c9baf595adbce6d5de6af" origin="Generated by Gradle"/>

‎x-pack/plugin/otel-data/build.gradle

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import com.google.protobuf.gradle.GenerateProtoTask
99
plugins {
1010
id 'elasticsearch.internal-es-plugin'
1111
id 'elasticsearch.internal-yaml-rest-test'
12+
id 'elasticsearch.internal-java-rest-test'
1213
id 'elasticsearch.internal-cluster-test'
1314
id('com.google.protobuf') version '0.9.5'
1415
}
@@ -47,10 +48,9 @@ dependencies {
4748
});
4849
opentelemetryProtobuf "open-telemetry:opentelemetry-proto:1.7.0@zip"
4950

51+
api project(":libs:exponential-histogram")
5052
compileOnly project(path: xpackModule('core'))
51-
testImplementation project(path: ':x-pack:plugin:stack')
5253
testImplementation(testArtifact(project(xpackModule('core'))))
53-
testImplementation project(':modules:data-streams')
5454
clusterModules project(':modules:data-streams')
5555
clusterModules project(':modules:ingest-common')
5656
clusterModules project(':modules:ingest-geoip')
@@ -66,6 +66,20 @@ dependencies {
6666
clusterModules project(xpackModule('wildcard'))
6767
clusterModules project(xpackModule('mapper-version'))
6868

69+
def otelVersion = "1.53.0"
70+
javaRestTestImplementation "io.opentelemetry:opentelemetry-api:$otelVersion"
71+
javaRestTestImplementation "io.opentelemetry:opentelemetry-common:$otelVersion"
72+
javaRestTestImplementation "io.opentelemetry:opentelemetry-context:$otelVersion"
73+
javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk:$otelVersion"
74+
javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-common:$otelVersion"
75+
javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-metrics:$otelVersion"
76+
javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-common:$otelVersion"
77+
javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp:$otelVersion"
78+
javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp-common:$otelVersion"
79+
javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion"
80+
javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion"
81+
javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion"
82+
6983
implementation "com.google.protobuf:protobuf-java:${protobufVersion}"
7084
// The protobuf plugin only adds a dependency for the variant relevant for the current platform.
7185
// Without explicitly adding all classifiers, the --write-verification-metadata task would only add the one for the current platform.
@@ -150,3 +164,7 @@ tasks.named("thirdPartyAudit").configure {
150164
tasks.named("licenseHeaders").configure {
151165
excludes << 'io/opentelemetry/proto/**/*'
152166
}
167+
168+
tasks.named("javaRestTest").configure {
169+
usesDefaultDistribution("Requires a bunch of xpack plugins")
170+
}

‎x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.action.otlp;
9+
10+
import io.opentelemetry.api.common.Attributes;
11+
import io.opentelemetry.exporter.internal.FailedExportException;
12+
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
13+
import io.opentelemetry.sdk.common.Clock;
14+
import io.opentelemetry.sdk.common.CompletableResultCode;
15+
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
16+
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
17+
import io.opentelemetry.sdk.metrics.data.MetricData;
18+
import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
19+
import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData;
20+
import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData;
21+
import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData;
22+
import io.opentelemetry.sdk.resources.Resource;
23+
24+
import org.elasticsearch.client.Request;
25+
import org.elasticsearch.common.settings.SecureString;
26+
import org.elasticsearch.common.settings.Settings;
27+
import org.elasticsearch.common.util.concurrent.ThreadContext;
28+
import org.elasticsearch.rest.RestStatus;
29+
import org.elasticsearch.test.cluster.ElasticsearchCluster;
30+
import org.elasticsearch.test.cluster.local.distribution.DistributionType;
31+
import org.elasticsearch.test.rest.ESRestTestCase;
32+
import org.junit.Before;
33+
import org.junit.ClassRule;
34+
35+
import java.time.Duration;
36+
import java.util.List;
37+
import java.util.concurrent.Executors;
38+
import java.util.concurrent.TimeUnit;
39+
40+
import static io.opentelemetry.api.common.AttributeKey.stringKey;
41+
import static org.hamcrest.Matchers.equalTo;
42+
import static org.hamcrest.Matchers.is;
43+
44+
public class OTLPMetricsIndexingRestIT extends ESRestTestCase {
45+
46+
private static final String USER = "test_admin";
47+
private static final String PASS = "x-pack-test-password";
48+
private static final Resource TEST_RESOURCE = Resource.create(Attributes.of(stringKey("service.name"), "elasticsearch"));
49+
private static final InstrumentationScopeInfo TEST_SCOPE = InstrumentationScopeInfo.create("io.opentelemetry.example.metrics");
50+
private OtlpHttpMetricExporter exporter;
51+
private SdkMeterProvider meterProvider;
52+
53+
@ClassRule
54+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
55+
.distribution(DistributionType.DEFAULT)
56+
.user(USER, PASS, "superuser", false)
57+
.setting("xpack.security.autoconfiguration.enabled", "false")
58+
.setting("xpack.license.self_generated.type", "trial")
59+
.build();
60+
61+
@Override
62+
protected String getTestRestCluster() {
63+
return cluster.getHttpAddresses();
64+
}
65+
66+
protected Settings restClientSettings() {
67+
String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
68+
return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build();
69+
}
70+
71+
@Before
72+
public void beforeTest() throws Exception {
73+
exporter = OtlpHttpMetricExporter.builder()
74+
.setEndpoint(getClusterHosts().getFirst().toURI() + "/_otlp/v1/metrics")
75+
.addHeader("Authorization", basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())))
76+
.build();
77+
meterProvider = SdkMeterProvider.builder()
78+
.registerMetricReader(
79+
PeriodicMetricReader.builder(exporter)
80+
.setExecutor(Executors.newScheduledThreadPool(0))
81+
.setInterval(Duration.ofNanos(Long.MAX_VALUE))
82+
.build()
83+
)
84+
.build();
85+
assertBusy(() -> assertOK(client().performRequest(new Request("GET", "_index_template/metrics-otel@template"))));
86+
boolean otlpEndpointEnabled = false;
87+
try {
88+
otlpEndpointEnabled = RestStatus.isSuccessful(
89+
client().performRequest(new Request("POST", "/_otlp/v1/metrics")).getStatusLine().getStatusCode()
90+
);
91+
} catch (Exception ignore) {}
92+
assumeTrue("Requires otlp_metrics feature flag to be enabled", otlpEndpointEnabled);
93+
}
94+
95+
@Override
96+
public void tearDown() throws Exception {
97+
meterProvider.close();
98+
super.tearDown();
99+
}
100+
101+
public void testIngestMetricDataViaMetricExporter() throws Exception {
102+
MetricData jvmMemoryMetricData = createDoubleGauge(
103+
TEST_RESOURCE,
104+
Attributes.empty(),
105+
"jvm.memory.total",
106+
Runtime.getRuntime().totalMemory(),
107+
"By",
108+
Clock.getDefault().now()
109+
);
110+
111+
FailedExportException.HttpExportException exception = assertThrows(
112+
FailedExportException.HttpExportException.class,
113+
() -> export(List.of(jvmMemoryMetricData))
114+
);
115+
assertThat(exception.getResponse().statusCode(), equalTo(RestStatus.NOT_IMPLEMENTED.getStatus()));
116+
}
117+
118+
private void export(List<MetricData> metrics) throws Exception {
119+
CompletableResultCode result = exporter.export(metrics).join(10, TimeUnit.SECONDS);
120+
Throwable failure = result.getFailureThrowable();
121+
if (failure instanceof Exception e) {
122+
throw e;
123+
} else if (failure != null) {
124+
throw new RuntimeException("Failed to export metrics", failure);
125+
}
126+
assertThat(result.isSuccess(), is(true));
127+
assertOK(client().performRequest(new Request("GET", "_refresh/metrics-*")));
128+
}
129+
130+
private static MetricData createDoubleGauge(
131+
Resource resource,
132+
Attributes attributes,
133+
String name,
134+
double value,
135+
String unit,
136+
long timeEpochNanos
137+
) {
138+
return ImmutableMetricData.createDoubleGauge(
139+
resource,
140+
TEST_SCOPE,
141+
name,
142+
"Your description could be here.",
143+
unit,
144+
ImmutableGaugeData.create(List.of(ImmutableDoublePointData.create(timeEpochNanos, timeEpochNanos, attributes, value)))
145+
);
146+
}
147+
}

‎x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java

Lines changed: 49 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,23 +10,31 @@
1010
import org.apache.logging.log4j.LogManager;
1111
import org.apache.logging.log4j.Logger;
1212
import org.apache.lucene.util.SetOnce;
13+
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
14+
import org.elasticsearch.cluster.node.DiscoveryNodes;
1315
import org.elasticsearch.cluster.service.ClusterService;
16+
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
17+
import org.elasticsearch.common.settings.ClusterSettings;
18+
import org.elasticsearch.common.settings.IndexScopedSettings;
1419
import org.elasticsearch.common.settings.Setting;
1520
import org.elasticsearch.common.settings.Settings;
21+
import org.elasticsearch.common.settings.SettingsFilter;
22+
import org.elasticsearch.common.util.FeatureFlag;
23+
import org.elasticsearch.features.NodeFeature;
1624
import org.elasticsearch.plugins.ActionPlugin;
1725
import org.elasticsearch.plugins.Plugin;
26+
import org.elasticsearch.rest.RestController;
27+
import org.elasticsearch.rest.RestHandler;
1828
import org.elasticsearch.xpack.core.XPackSettings;
29+
import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsRestAction;
30+
import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsTransportAction;
1931

2032
import java.util.Collection;
21-
import java.util.Collections;
2233
import java.util.List;
34+
import java.util.function.Predicate;
35+
import java.util.function.Supplier;
2336

2437
public class OTelPlugin extends Plugin implements ActionPlugin {
25-
private static final Logger logger = LogManager.getLogger(OTelPlugin.class);
26-
27-
final SetOnce<OTelIndexTemplateRegistry> registry = new SetOnce<>();
28-
29-
private final boolean enabled;
3038

3139
// OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry.
3240
//
@@ -38,10 +46,35 @@ public class OTelPlugin extends Plugin implements ActionPlugin {
3846
Setting.Property.Dynamic
3947
);
4048

49+
private static final Logger logger = LogManager.getLogger(OTelPlugin.class);
50+
51+
private static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled();
52+
private final SetOnce<OTelIndexTemplateRegistry> registry = new SetOnce<>();
53+
private final boolean enabled;
54+
4155
public OTelPlugin(Settings settings) {
4256
this.enabled = XPackSettings.OTEL_DATA_ENABLED.get(settings);
4357
}
4458

59+
@Override
60+
public Collection<RestHandler> getRestHandlers(
61+
Settings settings,
62+
NamedWriteableRegistry namedWriteableRegistry,
63+
RestController restController,
64+
ClusterSettings clusterSettings,
65+
IndexScopedSettings indexScopedSettings,
66+
SettingsFilter settingsFilter,
67+
IndexNameExpressionResolver indexNameExpressionResolver,
68+
Supplier<DiscoveryNodes> nodesInCluster,
69+
Predicate<NodeFeature> clusterSupportsFeature
70+
) {
71+
if (OTLP_METRICS_ENABLED) {
72+
return List.of(new OTLPMetricsRestAction());
73+
} else {
74+
return List.of();
75+
}
76+
}
77+
4578
@Override
4679
public Collection<?> createComponents(PluginServices services) {
4780
logger.info("OTel ingest plugin is {}", enabled ? "enabled" : "disabled");
@@ -55,7 +88,7 @@ public Collection<?> createComponents(PluginServices services) {
5588
registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings));
5689
registryInstance.initialize();
5790
}
58-
return Collections.emptyList();
91+
return List.of();
5992
}
6093

6194
@Override
@@ -67,4 +100,13 @@ public void close() {
67100
public List<Setting<?>> getSettings() {
68101
return List.of(OTEL_DATA_REGISTRY_ENABLED);
69102
}
103+
104+
@Override
105+
public Collection<ActionHandler> getActions() {
106+
if (OTLP_METRICS_ENABLED) {
107+
return List.of(new ActionHandler(OTLPMetricsTransportAction.TYPE, OTLPMetricsTransportAction.class));
108+
} else {
109+
return List.of();
110+
}
111+
}
70112
}

0 commit comments

Comments
 (0)