From 952608d251a10f6959b887fabf518593b0980ff8 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Thu, 21 Aug 2025 10:24:37 +0200 Subject: [PATCH 1/8] Add metrics OTLP endpoint skeleton --- .../resources/checkstyle_suppressions.xml | 1 + dev-tools/protoc_exe_sha256.sh | 56 +++++ gradle/verification-metadata.xml | 117 ++++++++++ .../test/ESSingleNodeTestCase.java | 28 ++- .../test/rest/ESRestTestCase.java | 2 +- x-pack/plugin/otel-data/build.gradle | 142 +++++++++++-- .../licenses/opentelemetry-LICENSE.txt | 201 ++++++++++++++++++ .../licenses/opentelemetry-NOTICE.txt | 0 .../otel-data/licenses/protobuf-LICENSE.txt | 32 +++ .../otel-data/licenses/protobuf-NOTICE.txt | 32 +++ .../action/otlp/OTLPMetricsIndexingIT.java | 183 ++++++++++++++++ .../xpack/oteldata/OTelPlugin.java | 56 ++++- .../oteldata/otlp/OTLPMetricsRestAction.java | 73 +++++++ .../otlp/OTLPMetricsTransportAction.java | 147 +++++++++++++ 14 files changed, 1041 insertions(+), 29 deletions(-) create mode 100644 dev-tools/protoc_exe_sha256.sh create mode 100644 x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt create mode 100644 x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt create mode 100644 x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt create mode 100644 x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt create mode 100644 x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java create mode 100644 x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java create mode 100644 x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index 98ded638773ce..cb4a58ee453a7 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -13,6 +13,7 @@ + diff --git a/dev-tools/protoc_exe_sha256.sh b/dev-tools/protoc_exe_sha256.sh new file mode 100644 index 0000000000000..58e31f4cdc50d --- /dev/null +++ b/dev-tools/protoc_exe_sha256.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the "Elastic License +# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +# Public License v 1"; you may not use this file except in compliance with, at +# your election, the "Elastic License 2.0", the "GNU Affero General Public +# License v3.0 only", or the "Server Side Public License, v 1". +# + +# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums +# URL to download from +VERSION="4.32.0" +URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/" +DOWNLOAD_DIR="protoc-${VERSION}-executables" + +# Create download directory if it doesn't exist +mkdir -p "${DOWNLOAD_DIR}" +cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; } + +# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.) +# Using grep with lookahead assertion to ensure we don't match .exe followed by something else +curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt + +if [ ! -s exe_files.txt ]; then + echo "No .exe files found at ${URL}" + exit 1 +fi + +echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..." + +# Download each file +while IFS= read -r file; do + curl -s -O "${URL}${file}" +done < exe_files.txt + +echo "Generating SHA256 checksums..." + +# Generate SHA256 checksums for all downloaded .exe files +if command -v shasum &> /dev/null; then + # macOS/some Linux + shasum -a 256 *.exe > SHA256SUMS.txt +elif command -v sha256sum &> /dev/null; then + # Most Linux distributions + sha256sum *.exe > SHA256SUMS.txt +else + echo "Neither shasum nor sha256sum command found. Cannot generate checksums." + exit 1 +fi + +# Print the checksums +cat SHA256SUMS.txt + +cd .. +rm -rf "${DOWNLOAD_DIR}" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8e3da59b925ad..b8e2b0705d52a 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -707,6 +707,11 @@ + + + + + @@ -867,6 +872,11 @@ + + + + + @@ -877,11 +887,53 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1709,6 +1761,16 @@ + + + + + + + + + + @@ -1724,11 +1786,61 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1874,6 +1986,11 @@ + + + + + diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 53214590e4a60..0040296b5656e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -19,6 +19,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.DeletePipelineRequest; @@ -75,6 +76,7 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -155,10 +157,28 @@ public void tearDown() throws Exception { throw e; } } - var deleteComposableIndexTemplateRequest = new TransportDeleteComposableIndexTemplateAction.Request("*"); - assertAcked(client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, deleteComposableIndexTemplateRequest).actionGet()); - var deleteComponentTemplateRequest = new TransportDeleteComponentTemplateAction.Request("*"); - assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, deleteComponentTemplateRequest).actionGet()); + var indexTemplates = client().execute( + GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*") + ).actionGet().indexTemplates(); + var deleteComposableIndexTemplateRequest = new TransportDeleteComposableIndexTemplateAction.Request( + indexTemplates.keySet().stream().filter(Predicate.not(ESRestTestCase::isXPackTemplate)).toArray(String[]::new) + ); + if (deleteComposableIndexTemplateRequest.names().length > 0) { + assertAcked( + client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, deleteComposableIndexTemplateRequest).actionGet() + ); + } + var componentTemplates = client().execute( + GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*") + ).actionGet().indexTemplates(); + var deleteComponentTemplateRequest = new TransportDeleteComponentTemplateAction.Request( + componentTemplates.keySet().stream().filter(Predicate.not(ESRestTestCase::isXPackTemplate)).toArray(String[]::new) + ); + if (deleteComponentTemplateRequest.names().length > 0) { + assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, deleteComponentTemplateRequest).actionGet()); + } assertAcked(indicesAdmin().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get()); Metadata metadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().getMetadata(); assertThat( diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 70dd46816d5d7..d008b55fc62b4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2283,7 +2283,7 @@ private static void assertAcked(String message, Response response) throws IOExce /** * Is this template one that is automatically created by xpack? */ - protected static boolean isXPackTemplate(String name) { + public static boolean isXPackTemplate(String name) { if (name.startsWith(".monitoring-")) { return true; } diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 0ae8022bff617..63b5041ff48af 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -4,9 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -apply plugin: 'elasticsearch.internal-es-plugin' -apply plugin: 'elasticsearch.internal-yaml-rest-test' -apply plugin: 'elasticsearch.internal-cluster-test' +plugins { + id 'elasticsearch.internal-es-plugin' + id 'elasticsearch.internal-yaml-rest-test' + id 'elasticsearch.internal-cluster-test' + id('com.google.protobuf') version '0.9.5' +} esplugin { name = 'x-pack-otel-data' @@ -15,23 +18,128 @@ esplugin { extendedPlugins = ['x-pack-core'] } +// manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version +def protobufVersion = "4.32.0" dependencies { + api project(":libs:exponential-histogram") compileOnly project(path: xpackModule('core')) testImplementation project(path: ':x-pack:plugin:stack') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(':modules:data-streams') - clusterModules project(':modules:data-streams') - clusterModules project(':modules:ingest-common') - clusterModules project(':modules:ingest-geoip') - clusterModules project(':modules:ingest-user-agent') - clusterModules project(':modules:lang-mustache') - clusterModules project(':modules:mapper-extras') - clusterModules project(xpackModule('analytics')) - clusterModules project(xpackModule('ilm')) - clusterModules project(xpackModule('mapper-aggregate-metric')) - clusterModules project(xpackModule('mapper-constant-keyword')) - clusterModules project(xpackModule('mapper-counted-keyword')) - clusterModules project(xpackModule('stack')) - clusterModules project(xpackModule('wildcard')) - clusterModules project(xpackModule('mapper-version')) + testImplementation project(':x-pack:plugin:esql') + testImplementation project(':x-pack:plugin:esql-core') + testImplementation project(':modules:data-streams') + testImplementation project(':modules:ingest-common') + testImplementation project(':modules:ingest-geoip') + testImplementation project(':modules:ingest-user-agent') + testImplementation project(':modules:lang-mustache') + testImplementation project(':modules:lang-painless') + testImplementation project(':modules:lang-painless:spi') + testImplementation project(':modules:mapper-extras') + testImplementation project(xpackModule('analytics')) + testImplementation project(xpackModule('ilm')) + testImplementation project(xpackModule('mapper-aggregate-metric')) + testImplementation project(xpackModule('mapper-constant-keyword')) + testImplementation project(xpackModule('mapper-counted-keyword')) + testImplementation project(xpackModule('stack')) + testImplementation project(xpackModule('wildcard')) + testImplementation project(xpackModule('mapper-version')) + + def otelVersion = "1.53.0" + testImplementation "io.opentelemetry:opentelemetry-api:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-common:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-context:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-sdk:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-sdk-common:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-sdk-metrics:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-exporter-common:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-exporter-otlp:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-exporter-otlp-common:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion" + testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion" + + implementation "com.google.protobuf:protobuf-java:${protobufVersion}" +} + +protobuf { + protoc { + // The artifact spec for the Protobuf Compiler + artifact = "com.google.protobuf:protoc:${protobufVersion}" + } +} + +def otlpProtoVersion = "1.7.0" // Version of the OpenTelemetry proto files to use + +// To generate the checksum, download the file and run "shasum -a 256 ~/path/to/vfoo.zip" +def protoChecksum = "ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc" +def protoArchivePath = layout.buildDirectory.file("archives/opentelemetry-proto-${otlpProtoVersion}.zip") + +def downloadProtoArchive = tasks.register("downloadProtoArchive") { + outputs.file(protoArchivePath) + onlyIf { !protoArchivePath.get().asFile.exists() } + doLast { + protoArchivePath.get().asFile.parentFile.mkdirs() + "https://github.com/open-telemetry/opentelemetry-proto/archive/v${otlpProtoVersion}.zip".toURL() + .withInputStream { is -> + protoArchivePath.get().asFile.withOutputStream {os -> os << is } + } + } +} + +def verifyProtoArchive = tasks.register("verifyProtoArchive") { + dependsOn(downloadProtoArchive) + doLast { + protoArchivePath.get().asFile.withInputStream { inputStream -> + def sha256 = inputStream.readAllBytes().digest("SHA-256") + if (sha256 != protoChecksum) { + throw new GradleException("Checksum verification failed for $protoArchivePath: expected $protoChecksum, got $sha256") + } + } + } +} + +def unzipProtoArchive = tasks.register("unzipProtoArchive", Copy) { + dependsOn(verifyProtoArchive) + from zipTree(protoArchivePath) + into layout.buildDirectory.dir("protos") +} + +// Avoid unnecessary String allocations in the generated AnyValue class +// We always need the ByteString (UTF-8) representation of string attributes +def adjustAnyValueBuilder = tasks.register("adjustAnyValueBuilder") { + doLast { + ant.replace( + file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java', + token: 'java.lang.String s = input.readStringRequireUtf8();', + value: 'com.google.protobuf.ByteString s = input.readBytes();', + encoding: 'UTF-8' + ) + } +} + +sourceSets { + main { + proto { + srcDir layout.buildDirectory.dir("protos/opentelemetry-proto-${otlpProtoVersion}") + } + } +} + +afterEvaluate { + tasks.named("generateProto") { + dependsOn(unzipProtoArchive) + finalizedBy(adjustAnyValueBuilder) + } +} + +idea { + module { + sourceDirs += layout.buildDirectory.dir("generated/sources/proto/main/java").get().asFile + } +} + +tasks.named("dependencyLicenses").configure { + mapping from: /opentelemetry-.*/, to: 'opentelemetry' + mapping from: /protobuf.*/, to: 'protobuf' } diff --git a/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt b/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt new file mode 100644 index 0000000000000..261eeb9e9f8b2 --- /dev/null +++ b/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt b/x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt b/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt b/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt new file mode 100644 index 0000000000000..19b305b00060a --- /dev/null +++ b/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt @@ -0,0 +1,32 @@ +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java b/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java new file mode 100644 index 0000000000000..b40e8cf44147b --- /dev/null +++ b/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java @@ -0,0 +1,183 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.action.otlp; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.exporter.internal.FailedExportException; +import io.opentelemetry.exporter.internal.grpc.GrpcExporterUtil; +import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; +import io.opentelemetry.sdk.common.Clock; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData; +import io.opentelemetry.sdk.resources.Resource; + +import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; +import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.transport.TransportAddress; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.http.HttpInfo; +import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; +import org.elasticsearch.ingest.common.IngestCommonPlugin; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; +import org.elasticsearch.xpack.analytics.AnalyticsPlugin; +import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin; +import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; +import org.elasticsearch.xpack.ilm.IndexLifecycle; +import org.elasticsearch.xpack.oteldata.OTelPlugin; +import org.elasticsearch.xpack.stack.StackPlugin; +import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; +import org.elasticsearch.xpack.wildcard.Wildcard; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static io.opentelemetry.api.common.AttributeKey.stringKey; +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; + +public class OTLPMetricsIndexingIT extends ESSingleNodeTestCase { + + private static final Resource TEST_RESOURCE = Resource.create(Attributes.of(stringKey("service.name"), "elasticsearch")); + private static final InstrumentationScopeInfo TEST_SCOPE = InstrumentationScopeInfo.create("io.opentelemetry.example.metrics"); + private OtlpHttpMetricExporter exporter; + private SdkMeterProvider meterProvider; + + @Override + protected Collection> getPlugins() { + return List.of( + DataStreamsPlugin.class, + InternalSettingsPlugin.class, + OTelPlugin.class, + StackPlugin.class, + EsqlPlugin.class, + VersionFieldPlugin.class, + CountedKeywordMapperPlugin.class, + ConstantKeywordMapperPlugin.class, + MapperExtrasPlugin.class, + Wildcard.class, + IndexLifecycle.class, + IngestCommonPlugin.class, + XPackPlugin.class, + PainlessPlugin.class, + AnalyticsPlugin.class, + AggregateMetricMapperPlugin.class + ); + } + + @Override + protected Settings nodeSettings() { + Settings.Builder newSettings = Settings.builder(); + newSettings.put(super.nodeSettings()); + // This essentially disables the automatic updates to end_time settings of a data stream's latest backing index. + newSettings.put(DataStreamsPlugin.TIME_SERIES_POLL_INTERVAL.getKey(), "10m"); + return newSettings.build(); + } + + @Override + protected boolean addMockHttpTransport() { + return false; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + exporter = OtlpHttpMetricExporter.builder().setEndpoint("http://localhost:" + getHttpPort() + "/_otlp/v1/metrics").build(); + meterProvider = SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(exporter) + .setExecutor(Executors.newScheduledThreadPool(0)) + .setInterval(Duration.ofNanos(Long.MAX_VALUE)) + .build() + ) + .build(); + assertBusy(() -> { + GetComposableIndexTemplateAction.Request getReq = new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*"); + var templates = client().execute(GetComposableIndexTemplateAction.INSTANCE, getReq).actionGet().indexTemplates(); + assertThat(templates, not(anEmptyMap())); + }); + } + + private int getHttpPort() { + NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get(); + assertFalse(nodesInfoResponse.hasFailures()); + assertEquals(1, nodesInfoResponse.getNodes().size()); + NodeInfo node = nodesInfoResponse.getNodes().getFirst(); + assertNotNull(node.getInfo(HttpInfo.class)); + TransportAddress publishAddress = node.getInfo(HttpInfo.class).address().publishAddress(); + InetSocketAddress address = publishAddress.address(); + return address.getPort(); + } + + @Override + public void tearDown() throws Exception { + meterProvider.close(); + super.tearDown(); + } + + public void testIngestMetricDataViaMetricExporter() throws Exception { + MetricData jvmMemoryMetricData = createDoubleGauge( + TEST_RESOURCE, + Attributes.empty(), + "jvm.memory.total", + Runtime.getRuntime().totalMemory(), + "By", + Clock.getDefault().now() + ); + + assertThrows("OTLP metrics is not implemented yet", RuntimeException.class, () -> export(List.of(jvmMemoryMetricData))); + } + + private void export(List metrics) throws IOException { + var result = exporter.export(metrics).join(10, TimeUnit.SECONDS); + Throwable failure = result.getFailureThrowable(); + if (failure instanceof FailedExportException.HttpExportException httpExportException) { + throw new RuntimeException(GrpcExporterUtil.getStatusMessage(httpExportException.getResponse().responseBody())); + } else if (failure != null) { + throw new RuntimeException("Failed to export metrics", failure); + } + assertThat(result.isSuccess(), is(true)); + admin().indices().prepareRefresh().execute().actionGet(); + } + + private static MetricData createDoubleGauge( + Resource resource, + Attributes attributes, + String name, + double value, + String unit, + long timeEpochNanos + ) { + return ImmutableMetricData.createDoubleGauge( + resource, + TEST_SCOPE, + name, + "Your description could be here.", + unit, + ImmutableGaugeData.create(List.of(ImmutableDoublePointData.create(timeEpochNanos, timeEpochNanos, attributes, value))) + ); + } +} diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 67bd8c4e002d3..8addef596014e 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -10,23 +10,31 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.util.FeatureFlag; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsRestAction; +import org.elasticsearch.xpack.oteldata.otlp.OTLPMetricsTransportAction; import java.util.Collection; -import java.util.Collections; import java.util.List; +import java.util.function.Predicate; +import java.util.function.Supplier; public class OTelPlugin extends Plugin implements ActionPlugin { - private static final Logger logger = LogManager.getLogger(OTelPlugin.class); - - final SetOnce registry = new SetOnce<>(); - - private final boolean enabled; // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry. // @@ -38,10 +46,35 @@ public class OTelPlugin extends Plugin implements ActionPlugin { Setting.Property.Dynamic ); + private static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled(); + private static final Logger logger = LogManager.getLogger(OTelPlugin.class); + + private final SetOnce registry = new SetOnce<>(); + private final boolean enabled; + public OTelPlugin(Settings settings) { this.enabled = XPackSettings.OTEL_DATA_ENABLED.get(settings); } + @Override + public Collection getRestHandlers( + Settings settings, + NamedWriteableRegistry namedWriteableRegistry, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster, + Predicate clusterSupportsFeature + ) { + if (OTLP_METRICS_ENABLED) { + return List.of(new OTLPMetricsRestAction()); + } else { + return List.of(); + } + } + @Override public Collection createComponents(PluginServices services) { logger.info("OTel ingest plugin is {}", enabled ? "enabled" : "disabled"); @@ -55,7 +88,7 @@ public Collection createComponents(PluginServices services) { registryInstance.setEnabled(OTEL_DATA_REGISTRY_ENABLED.get(settings)); registryInstance.initialize(); } - return Collections.emptyList(); + return List.of(); } @Override @@ -67,4 +100,13 @@ public void close() { public List> getSettings() { return List.of(OTEL_DATA_REGISTRY_ENABLED); } + + @Override + public Collection getActions() { + if (OTLP_METRICS_ENABLED) { + return List.of(new ActionHandler(OTLPMetricsTransportAction.TYPE, OTLPMetricsTransportAction.class)); + } else { + return List.of(); + } + } } diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java new file mode 100644 index 0000000000000..0d2135009c922 --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.oteldata.otlp; + +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestResponseListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class OTLPMetricsRestAction extends BaseRestHandler { + @Override + public String getName() { + return "otlp_metrics_action"; + } + + @Override + public List routes() { + return List.of(new Route(POST, "/_otlp/v1/metrics")); + } + + @Override + public boolean mediaTypesValid(RestRequest request) { + return request.getXContentType() == null + && request.getParsedContentType().mediaTypeWithoutParameters().equals("application/x-protobuf"); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { + if (request.hasContent()) { + var transportRequest = new OTLPMetricsTransportAction.MetricsRequest(request.content().retain()); + return channel -> client.execute( + OTLPMetricsTransportAction.TYPE, + transportRequest, + ActionListener.releaseBefore(request.content(), new RestResponseListener<>(channel) { + @Override + public RestResponse buildResponse(OTLPMetricsTransportAction.MetricsResponse r) { + RestStatus restStatus = r.getStatus(); + return new RestResponse(restStatus, "application/x-protobuf", r.getResponse()); + } + }) + ); + } + + // If the server receives an empty request + // (a request that does not carry any telemetry data) + // the server SHOULD respond with success. + // https://opentelemetry.io/docs/specs/otlp/#full-success-1 + return channel -> channel.sendResponse( + new RestResponse( + RestStatus.OK, + "application/x-protobuf", + new BytesArray(ExportMetricsServiceResponse.newBuilder().build().toByteArray()) + ) + ); + } + +} diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java new file mode 100644 index 0000000000000..7697d3a900b5e --- /dev/null +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.oteldata.otlp; + +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsPartialSuccess; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; + +import com.google.protobuf.MessageLite; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.injection.guice.Inject; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; + +/** + * Transport action for handling OpenTelemetry Protocol (OTLP) Metrics requests. + * This action processes the incoming metrics data, groups data points, and invokes the + * appropriate Elasticsearch bulk indexing operations to store the metrics. + * It also handles the response according to the OpenTelemetry Protocol specifications, + * including success, partial success responses, and errors due to bad data or server errors. + * + * @see OTLP Specification + */ +public class OTLPMetricsTransportAction extends HandledTransportAction< + OTLPMetricsTransportAction.MetricsRequest, + OTLPMetricsTransportAction.MetricsResponse> { + + public static final String NAME = "indices:data/write/metrics"; + public static final ActionType TYPE = new ActionType<>(NAME); + + private static final Logger logger = LogManager.getLogger(OTLPMetricsTransportAction.class); + private final Client client; + + @Inject + public OTLPMetricsTransportAction( + TransportService transportService, + ActionFilters actionFilters, + ThreadPool threadPool, + Client client + ) { + super(NAME, transportService, actionFilters, MetricsRequest::new, threadPool.executor(ThreadPool.Names.WRITE)); + this.client = client; + } + + @Override + protected void doExecute(Task task, MetricsRequest request, ActionListener listener) { + try { + var metricsServiceRequest = ExportMetricsServiceRequest.parseFrom(request.exportMetricsServiceRequest.streamInput()); + + listener.onResponse( + new MetricsResponse( + RestStatus.NOT_IMPLEMENTED, + responseWithRejectedDataPoints( + metricsServiceRequest.getResourceMetricsList().size(), + "OTLP metrics is not implemented yet" + ) + ) + ); + + } catch (Exception e) { + logger.error("failed to execute otlp metrics request", e); + + listener.onResponse( + new MetricsResponse( + RestStatus.INTERNAL_SERVER_ERROR, + ExportMetricsServiceResponse.newBuilder() + .getPartialSuccessBuilder() + .setErrorMessage("unexpected error while processing otlp metrics request: " + e.getMessage()) + .build() + ) + ); + } + } + + private static ExportMetricsPartialSuccess responseWithRejectedDataPoints(int rejectedDataPoints, String message) { + return ExportMetricsServiceResponse.newBuilder() + .getPartialSuccessBuilder() + .setRejectedDataPoints(rejectedDataPoints) + .setErrorMessage(message) + .build(); + } + + public static class MetricsRequest extends ActionRequest { + private final BytesReference exportMetricsServiceRequest; + + public MetricsRequest(StreamInput in) throws IOException { + super(in); + exportMetricsServiceRequest = in.readBytesReference(); + } + + public MetricsRequest(BytesReference exportMetricsServiceRequest) { + this.exportMetricsServiceRequest = exportMetricsServiceRequest; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + } + + public static class MetricsResponse extends ActionResponse { + private final BytesReference response; + private final RestStatus status; + + public MetricsResponse(RestStatus status, MessageLite response) { + this.response = new BytesArray(response.toByteArray()); + this.status = status; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeBytesReference(response); + out.writeEnum(status); + } + + public BytesReference getResponse() { + return response; + } + + public RestStatus getStatus() { + return status; + } + } +} From 1b2b7b31d6503bb83e63d3b9adaa51dce001537d Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Thu, 21 Aug 2025 11:24:01 +0200 Subject: [PATCH 2/8] Remove protobuf for now Protobuf will be added in a separate PR to keep the scope more contained --- x-pack/plugin/otel-data/build.gradle | 87 -------- .../licenses/opentelemetry-LICENSE.txt | 201 ------------------ .../licenses/opentelemetry-NOTICE.txt | 0 .../otel-data/licenses/protobuf-LICENSE.txt | 32 --- .../otel-data/licenses/protobuf-NOTICE.txt | 32 --- .../oteldata/otlp/OTLPMetricsRestAction.java | 10 +- .../otlp/OTLPMetricsTransportAction.java | 45 +--- 7 files changed, 4 insertions(+), 403 deletions(-) delete mode 100644 x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt delete mode 100644 x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt delete mode 100644 x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt delete mode 100644 x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 63b5041ff48af..344753ed1b93b 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -8,7 +8,6 @@ plugins { id 'elasticsearch.internal-es-plugin' id 'elasticsearch.internal-yaml-rest-test' id 'elasticsearch.internal-cluster-test' - id('com.google.protobuf') version '0.9.5' } esplugin { @@ -18,8 +17,6 @@ esplugin { extendedPlugins = ['x-pack-core'] } -// manually update verification-metadata.xml via dev-tools/protoc_exe_sha256.sh when updating the protobuf version -def protobufVersion = "4.32.0" dependencies { api project(":libs:exponential-histogram") compileOnly project(path: xpackModule('core')) @@ -58,88 +55,4 @@ dependencies { testImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion" testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion" testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion" - - implementation "com.google.protobuf:protobuf-java:${protobufVersion}" -} - -protobuf { - protoc { - // The artifact spec for the Protobuf Compiler - artifact = "com.google.protobuf:protoc:${protobufVersion}" - } -} - -def otlpProtoVersion = "1.7.0" // Version of the OpenTelemetry proto files to use - -// To generate the checksum, download the file and run "shasum -a 256 ~/path/to/vfoo.zip" -def protoChecksum = "ddb80357ff146f5e3bda584907185b1f635412a4b31edf6f96b102a18b8e05dc" -def protoArchivePath = layout.buildDirectory.file("archives/opentelemetry-proto-${otlpProtoVersion}.zip") - -def downloadProtoArchive = tasks.register("downloadProtoArchive") { - outputs.file(protoArchivePath) - onlyIf { !protoArchivePath.get().asFile.exists() } - doLast { - protoArchivePath.get().asFile.parentFile.mkdirs() - "https://github.com/open-telemetry/opentelemetry-proto/archive/v${otlpProtoVersion}.zip".toURL() - .withInputStream { is -> - protoArchivePath.get().asFile.withOutputStream {os -> os << is } - } - } -} - -def verifyProtoArchive = tasks.register("verifyProtoArchive") { - dependsOn(downloadProtoArchive) - doLast { - protoArchivePath.get().asFile.withInputStream { inputStream -> - def sha256 = inputStream.readAllBytes().digest("SHA-256") - if (sha256 != protoChecksum) { - throw new GradleException("Checksum verification failed for $protoArchivePath: expected $protoChecksum, got $sha256") - } - } - } -} - -def unzipProtoArchive = tasks.register("unzipProtoArchive", Copy) { - dependsOn(verifyProtoArchive) - from zipTree(protoArchivePath) - into layout.buildDirectory.dir("protos") -} - -// Avoid unnecessary String allocations in the generated AnyValue class -// We always need the ByteString (UTF-8) representation of string attributes -def adjustAnyValueBuilder = tasks.register("adjustAnyValueBuilder") { - doLast { - ant.replace( - file: 'build/generated/sources/proto/main/java/io/opentelemetry/proto/common/v1/AnyValue.java', - token: 'java.lang.String s = input.readStringRequireUtf8();', - value: 'com.google.protobuf.ByteString s = input.readBytes();', - encoding: 'UTF-8' - ) - } -} - -sourceSets { - main { - proto { - srcDir layout.buildDirectory.dir("protos/opentelemetry-proto-${otlpProtoVersion}") - } - } -} - -afterEvaluate { - tasks.named("generateProto") { - dependsOn(unzipProtoArchive) - finalizedBy(adjustAnyValueBuilder) - } -} - -idea { - module { - sourceDirs += layout.buildDirectory.dir("generated/sources/proto/main/java").get().asFile - } -} - -tasks.named("dependencyLicenses").configure { - mapping from: /opentelemetry-.*/, to: 'opentelemetry' - mapping from: /protobuf.*/, to: 'protobuf' } diff --git a/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt b/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt deleted file mode 100644 index 261eeb9e9f8b2..0000000000000 --- a/x-pack/plugin/otel-data/licenses/opentelemetry-LICENSE.txt +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt b/x-pack/plugin/otel-data/licenses/opentelemetry-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt b/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt deleted file mode 100644 index 19b305b00060a..0000000000000 --- a/x-pack/plugin/otel-data/licenses/protobuf-LICENSE.txt +++ /dev/null @@ -1,32 +0,0 @@ -Copyright 2008 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. diff --git a/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt b/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt deleted file mode 100644 index 19b305b00060a..0000000000000 --- a/x-pack/plugin/otel-data/licenses/protobuf-NOTICE.txt +++ /dev/null @@ -1,32 +0,0 @@ -Copyright 2008 Google Inc. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Code generated by the Protocol Buffer compiler is owned by the owner -of the input file used when generating it. This code is not -standalone and requires a support library to be linked with it. This -support library is itself covered by the above license. diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java index 0d2135009c922..23f46363d3060 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsRestAction.java @@ -7,8 +7,6 @@ package org.elasticsearch.xpack.oteldata.otlp; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; - import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.bytes.BytesArray; @@ -61,13 +59,7 @@ public RestResponse buildResponse(OTLPMetricsTransportAction.MetricsResponse r) // (a request that does not carry any telemetry data) // the server SHOULD respond with success. // https://opentelemetry.io/docs/specs/otlp/#full-success-1 - return channel -> channel.sendResponse( - new RestResponse( - RestStatus.OK, - "application/x-protobuf", - new BytesArray(ExportMetricsServiceResponse.newBuilder().build().toByteArray()) - ) - ); + return channel -> channel.sendResponse(new RestResponse(RestStatus.OK, "application/x-protobuf", new BytesArray(new byte[0]))); } } diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java index 7697d3a900b5e..0a9d646d1d81a 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java @@ -7,12 +7,6 @@ package org.elasticsearch.xpack.oteldata.otlp; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsPartialSuccess; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; -import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; - -import com.google.protobuf.MessageLite; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; @@ -67,40 +61,7 @@ public OTLPMetricsTransportAction( @Override protected void doExecute(Task task, MetricsRequest request, ActionListener listener) { - try { - var metricsServiceRequest = ExportMetricsServiceRequest.parseFrom(request.exportMetricsServiceRequest.streamInput()); - - listener.onResponse( - new MetricsResponse( - RestStatus.NOT_IMPLEMENTED, - responseWithRejectedDataPoints( - metricsServiceRequest.getResourceMetricsList().size(), - "OTLP metrics is not implemented yet" - ) - ) - ); - - } catch (Exception e) { - logger.error("failed to execute otlp metrics request", e); - - listener.onResponse( - new MetricsResponse( - RestStatus.INTERNAL_SERVER_ERROR, - ExportMetricsServiceResponse.newBuilder() - .getPartialSuccessBuilder() - .setErrorMessage("unexpected error while processing otlp metrics request: " + e.getMessage()) - .build() - ) - ); - } - } - - private static ExportMetricsPartialSuccess responseWithRejectedDataPoints(int rejectedDataPoints, String message) { - return ExportMetricsServiceResponse.newBuilder() - .getPartialSuccessBuilder() - .setRejectedDataPoints(rejectedDataPoints) - .setErrorMessage(message) - .build(); + listener.onResponse(new MetricsResponse(RestStatus.NOT_IMPLEMENTED, new BytesArray(new byte[0]))); } public static class MetricsRequest extends ActionRequest { @@ -125,8 +86,8 @@ public static class MetricsResponse extends ActionResponse { private final BytesReference response; private final RestStatus status; - public MetricsResponse(RestStatus status, MessageLite response) { - this.response = new BytesArray(response.toByteArray()); + public MetricsResponse(RestStatus status, BytesReference response) { + this.response = response; this.status = status; } From 72fc5dd66dc5252e7992ecd7893ddf243aa1f5e9 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Thu, 21 Aug 2025 11:31:28 +0200 Subject: [PATCH 3/8] Remove more proto stuff --- .../resources/checkstyle_suppressions.xml | 1 - dev-tools/protoc_exe_sha256.sh | 56 ------------------ gradle/verification-metadata.xml | 57 ------------------- 3 files changed, 114 deletions(-) delete mode 100644 dev-tools/protoc_exe_sha256.sh diff --git a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml index cb4a58ee453a7..98ded638773ce 100644 --- a/build-tools-internal/src/main/resources/checkstyle_suppressions.xml +++ b/build-tools-internal/src/main/resources/checkstyle_suppressions.xml @@ -13,7 +13,6 @@ - diff --git a/dev-tools/protoc_exe_sha256.sh b/dev-tools/protoc_exe_sha256.sh deleted file mode 100644 index 58e31f4cdc50d..0000000000000 --- a/dev-tools/protoc_exe_sha256.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the "Elastic License -# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side -# Public License v 1"; you may not use this file except in compliance with, at -# your election, the "Elastic License 2.0", the "GNU Affero General Public -# License v3.0 only", or the "Server Side Public License, v 1". -# - -# Script to download all .exe files from protobuf protoc repository and generate SHA256 checksums -# URL to download from -VERSION="4.32.0" -URL="https://repo1.maven.org/maven2/com/google/protobuf/protoc/${VERSION}/" -DOWNLOAD_DIR="protoc-${VERSION}-executables" - -# Create download directory if it doesn't exist -mkdir -p "${DOWNLOAD_DIR}" -cd "${DOWNLOAD_DIR}" || { echo "Failed to create/enter download directory"; exit 1; } - -# Get the HTML content, extract links to .exe files (but not .exe.md5 etc.) -# Using grep with lookahead assertion to ensure we don't match .exe followed by something else -curl -s "${URL}" | grep -o 'href="[^"]*\.exe"' | grep -v -E 'jsonl' | grep -v -E '\.exe\.[^"]+' | sed 's/href="//g' | sed 's/"//g' > exe_files.txt - -if [ ! -s exe_files.txt ]; then - echo "No .exe files found at ${URL}" - exit 1 -fi - -echo "Found $(wc -l < exe_files.txt | tr -d ' ') .exe files. Downloading..." - -# Download each file -while IFS= read -r file; do - curl -s -O "${URL}${file}" -done < exe_files.txt - -echo "Generating SHA256 checksums..." - -# Generate SHA256 checksums for all downloaded .exe files -if command -v shasum &> /dev/null; then - # macOS/some Linux - shasum -a 256 *.exe > SHA256SUMS.txt -elif command -v sha256sum &> /dev/null; then - # Most Linux distributions - sha256sum *.exe > SHA256SUMS.txt -else - echo "Neither shasum nor sha256sum command found. Cannot generate checksums." - exit 1 -fi - -# Print the checksums -cat SHA256SUMS.txt - -cd .. -rm -rf "${DOWNLOAD_DIR}" diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 0817b2b0486ed..36d8b5fed7cbf 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -707,11 +707,6 @@ - - - - - @@ -872,11 +867,6 @@ - - - - - @@ -887,53 +877,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1986,11 +1934,6 @@ - - - - - From 4d093dfd322e6247981b4e06d958413022fdfe3f Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 11:37:13 +0200 Subject: [PATCH 4/8] Fix test dependency declaration --- x-pack/plugin/otel-data/build.gradle | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 344753ed1b93b..2cdd21d7dcee8 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -20,8 +20,23 @@ esplugin { dependencies { api project(":libs:exponential-histogram") compileOnly project(path: xpackModule('core')) - testImplementation project(path: ':x-pack:plugin:stack') testImplementation(testArtifact(project(xpackModule('core')))) + // for yamlRestTest + clusterModules project(':modules:data-streams') + clusterModules project(':modules:ingest-common') + clusterModules project(':modules:ingest-geoip') + clusterModules project(':modules:ingest-user-agent') + clusterModules project(':modules:lang-mustache') + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('analytics')) + clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-aggregate-metric')) + clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(xpackModule('mapper-counted-keyword')) + clusterModules project(xpackModule('stack')) + clusterModules project(xpackModule('wildcard')) + clusterModules project(xpackModule('mapper-version')) + // for internalClusterTest testImplementation project(':modules:data-streams') testImplementation project(':x-pack:plugin:esql') testImplementation project(':x-pack:plugin:esql-core') From 342373e79c05ac0e7741be63c138b5d3a9cc3f29 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 22 Aug 2025 15:26:47 +0200 Subject: [PATCH 5/8] Fix action test --- .../xpack/oteldata/otlp/OTLPMetricsTransportAction.java | 2 +- .../org/elasticsearch/xpack/security/operator/Constants.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java index 0a9d646d1d81a..2b7e83540f73a 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java @@ -42,7 +42,7 @@ public class OTLPMetricsTransportAction extends HandledTransportAction< OTLPMetricsTransportAction.MetricsRequest, OTLPMetricsTransportAction.MetricsResponse> { - public static final String NAME = "indices:data/write/metrics"; + public static final String NAME = "indices:data/write/otlp/metrics"; public static final ActionType TYPE = new ActionType<>(NAME); private static final Logger logger = LogManager.getLogger(OTLPMetricsTransportAction.class); diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 233c7278eed98..ff6a0aaad196d 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -609,6 +609,7 @@ public class Constants { "indices:data/write/delete", "indices:data/write/delete/byquery", "indices:data/write/index", + "indices:data/write/otlp/metrics", "indices:data/write/reindex", "indices:data/write/update", "indices:data/write/update/byquery", From 57ea866a88b2d4f24a47ae11b197712619ffdc29 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 27 Aug 2025 12:23:14 +0200 Subject: [PATCH 6/8] Skip tests when feature flag is disabled --- .../org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java | 1 + .../main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java b/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java index b40e8cf44147b..b737123fc75d0 100644 --- a/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java +++ b/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java @@ -119,6 +119,7 @@ public void setUp() throws Exception { var templates = client().execute(GetComposableIndexTemplateAction.INSTANCE, getReq).actionGet().indexTemplates(); assertThat(templates, not(anEmptyMap())); }); + assumeTrue("Requires otlp_metrics feature flag to be enabled", OTelPlugin.OTLP_METRICS_ENABLED); } private int getHttpPort() { diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 8addef596014e..72e1d6006bfa9 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -36,6 +36,8 @@ public class OTelPlugin extends Plugin implements ActionPlugin { + public static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled(); + // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry. // // This setting will be ignored if the plugin is disabled. @@ -46,7 +48,6 @@ public class OTelPlugin extends Plugin implements ActionPlugin { Setting.Property.Dynamic ); - private static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled(); private static final Logger logger = LogManager.getLogger(OTelPlugin.class); private final SetOnce registry = new SetOnce<>(); From 35269937941fc7deb7170cf60d1c64d791c4831c Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Wed, 27 Aug 2025 15:58:52 +0200 Subject: [PATCH 7/8] Convert to java rest test --- .../test/ESSingleNodeTestCase.java | 28 +-- .../test/rest/ESRestTestCase.java | 2 +- x-pack/plugin/otel-data/build.gradle | 50 ++--- .../action/otlp/OTLPMetricsIndexingIT.java | 184 ------------------ .../otlp/OTLPMetricsIndexingRestIT.java | 146 ++++++++++++++ .../xpack/oteldata/OTelPlugin.java | 3 +- .../otlp/OTLPMetricsTransportAction.java | 3 +- .../xpack/security/authz/RBACEngine.java | 1 + 8 files changed, 172 insertions(+), 245 deletions(-) delete mode 100644 x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java create mode 100644 x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 0040296b5656e..53214590e4a60 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -19,7 +19,6 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; -import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; import org.elasticsearch.action.datastreams.DeleteDataStreamAction; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.ingest.DeletePipelineRequest; @@ -76,7 +75,6 @@ import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -157,28 +155,10 @@ public void tearDown() throws Exception { throw e; } } - var indexTemplates = client().execute( - GetComposableIndexTemplateAction.INSTANCE, - new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*") - ).actionGet().indexTemplates(); - var deleteComposableIndexTemplateRequest = new TransportDeleteComposableIndexTemplateAction.Request( - indexTemplates.keySet().stream().filter(Predicate.not(ESRestTestCase::isXPackTemplate)).toArray(String[]::new) - ); - if (deleteComposableIndexTemplateRequest.names().length > 0) { - assertAcked( - client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, deleteComposableIndexTemplateRequest).actionGet() - ); - } - var componentTemplates = client().execute( - GetComposableIndexTemplateAction.INSTANCE, - new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*") - ).actionGet().indexTemplates(); - var deleteComponentTemplateRequest = new TransportDeleteComponentTemplateAction.Request( - componentTemplates.keySet().stream().filter(Predicate.not(ESRestTestCase::isXPackTemplate)).toArray(String[]::new) - ); - if (deleteComponentTemplateRequest.names().length > 0) { - assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, deleteComponentTemplateRequest).actionGet()); - } + var deleteComposableIndexTemplateRequest = new TransportDeleteComposableIndexTemplateAction.Request("*"); + assertAcked(client().execute(TransportDeleteComposableIndexTemplateAction.TYPE, deleteComposableIndexTemplateRequest).actionGet()); + var deleteComponentTemplateRequest = new TransportDeleteComponentTemplateAction.Request("*"); + assertAcked(client().execute(TransportDeleteComponentTemplateAction.TYPE, deleteComponentTemplateRequest).actionGet()); assertAcked(indicesAdmin().prepareDelete("*").setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN).get()); Metadata metadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().getMetadata(); assertThat( diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index d008b55fc62b4..70dd46816d5d7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2283,7 +2283,7 @@ private static void assertAcked(String message, Response response) throws IOExce /** * Is this template one that is automatically created by xpack? */ - public static boolean isXPackTemplate(String name) { + protected static boolean isXPackTemplate(String name) { if (name.startsWith(".monitoring-")) { return true; } diff --git a/x-pack/plugin/otel-data/build.gradle b/x-pack/plugin/otel-data/build.gradle index 25a162aba0971..263aa5674c656 100644 --- a/x-pack/plugin/otel-data/build.gradle +++ b/x-pack/plugin/otel-data/build.gradle @@ -9,6 +9,7 @@ import com.google.protobuf.gradle.GenerateProtoTask plugins { id 'elasticsearch.internal-es-plugin' id 'elasticsearch.internal-yaml-rest-test' + id 'elasticsearch.internal-java-rest-test' id 'elasticsearch.internal-cluster-test' id('com.google.protobuf') version '0.9.5' } @@ -50,7 +51,6 @@ dependencies { api project(":libs:exponential-histogram") compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) - // for yamlRestTest clusterModules project(':modules:data-streams') clusterModules project(':modules:ingest-common') clusterModules project(':modules:ingest-geoip') @@ -65,40 +65,20 @@ dependencies { clusterModules project(xpackModule('stack')) clusterModules project(xpackModule('wildcard')) clusterModules project(xpackModule('mapper-version')) - // for internalClusterTest - testImplementation project(':modules:data-streams') - testImplementation project(':x-pack:plugin:esql') - testImplementation project(':x-pack:plugin:esql-core') - testImplementation project(':modules:data-streams') - testImplementation project(':modules:ingest-common') - testImplementation project(':modules:ingest-geoip') - testImplementation project(':modules:ingest-user-agent') - testImplementation project(':modules:lang-mustache') - testImplementation project(':modules:lang-painless') - testImplementation project(':modules:lang-painless:spi') - testImplementation project(':modules:mapper-extras') - testImplementation project(xpackModule('analytics')) - testImplementation project(xpackModule('ilm')) - testImplementation project(xpackModule('mapper-aggregate-metric')) - testImplementation project(xpackModule('mapper-constant-keyword')) - testImplementation project(xpackModule('mapper-counted-keyword')) - testImplementation project(xpackModule('stack')) - testImplementation project(xpackModule('wildcard')) - testImplementation project(xpackModule('mapper-version')) def otelVersion = "1.53.0" - testImplementation "io.opentelemetry:opentelemetry-api:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-common:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-context:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-sdk:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-sdk-common:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-sdk-metrics:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-exporter-common:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-exporter-otlp:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-exporter-otlp-common:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion" - testImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-api:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-common:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-context:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-common:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-metrics:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-common:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-otlp-common:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-exporter-sender-jdk:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:$otelVersion" + javaRestTestImplementation "io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:$otelVersion" implementation "com.google.protobuf:protobuf-java:${protobufVersion}" // The protobuf plugin only adds a dependency for the variant relevant for the current platform. @@ -184,3 +164,7 @@ tasks.named("thirdPartyAudit").configure { tasks.named("licenseHeaders").configure { excludes << 'io/opentelemetry/proto/**/*' } + +tasks.named("javaRestTest").configure { + usesDefaultDistribution("Requires a bunch of xpack plugins") +} diff --git a/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java b/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java deleted file mode 100644 index b737123fc75d0..0000000000000 --- a/x-pack/plugin/otel-data/src/internalClusterTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingIT.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.action.otlp; - -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.exporter.internal.FailedExportException; -import io.opentelemetry.exporter.internal.grpc.GrpcExporterUtil; -import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; -import io.opentelemetry.sdk.common.Clock; -import io.opentelemetry.sdk.common.InstrumentationScopeInfo; -import io.opentelemetry.sdk.metrics.SdkMeterProvider; -import io.opentelemetry.sdk.metrics.data.MetricData; -import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; -import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData; -import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData; -import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData; -import io.opentelemetry.sdk.resources.Resource; - -import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; -import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; -import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.datastreams.DataStreamsPlugin; -import org.elasticsearch.http.HttpInfo; -import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin; -import org.elasticsearch.ingest.common.IngestCommonPlugin; -import org.elasticsearch.painless.PainlessPlugin; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.xpack.aggregatemetric.AggregateMetricMapperPlugin; -import org.elasticsearch.xpack.analytics.AnalyticsPlugin; -import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; -import org.elasticsearch.xpack.core.XPackPlugin; -import org.elasticsearch.xpack.countedkeyword.CountedKeywordMapperPlugin; -import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; -import org.elasticsearch.xpack.ilm.IndexLifecycle; -import org.elasticsearch.xpack.oteldata.OTelPlugin; -import org.elasticsearch.xpack.stack.StackPlugin; -import org.elasticsearch.xpack.versionfield.VersionFieldPlugin; -import org.elasticsearch.xpack.wildcard.Wildcard; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.time.Duration; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static io.opentelemetry.api.common.AttributeKey.stringKey; -import static org.hamcrest.Matchers.anEmptyMap; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; - -public class OTLPMetricsIndexingIT extends ESSingleNodeTestCase { - - private static final Resource TEST_RESOURCE = Resource.create(Attributes.of(stringKey("service.name"), "elasticsearch")); - private static final InstrumentationScopeInfo TEST_SCOPE = InstrumentationScopeInfo.create("io.opentelemetry.example.metrics"); - private OtlpHttpMetricExporter exporter; - private SdkMeterProvider meterProvider; - - @Override - protected Collection> getPlugins() { - return List.of( - DataStreamsPlugin.class, - InternalSettingsPlugin.class, - OTelPlugin.class, - StackPlugin.class, - EsqlPlugin.class, - VersionFieldPlugin.class, - CountedKeywordMapperPlugin.class, - ConstantKeywordMapperPlugin.class, - MapperExtrasPlugin.class, - Wildcard.class, - IndexLifecycle.class, - IngestCommonPlugin.class, - XPackPlugin.class, - PainlessPlugin.class, - AnalyticsPlugin.class, - AggregateMetricMapperPlugin.class - ); - } - - @Override - protected Settings nodeSettings() { - Settings.Builder newSettings = Settings.builder(); - newSettings.put(super.nodeSettings()); - // This essentially disables the automatic updates to end_time settings of a data stream's latest backing index. - newSettings.put(DataStreamsPlugin.TIME_SERIES_POLL_INTERVAL.getKey(), "10m"); - return newSettings.build(); - } - - @Override - protected boolean addMockHttpTransport() { - return false; - } - - @Override - public void setUp() throws Exception { - super.setUp(); - exporter = OtlpHttpMetricExporter.builder().setEndpoint("http://localhost:" + getHttpPort() + "/_otlp/v1/metrics").build(); - meterProvider = SdkMeterProvider.builder() - .registerMetricReader( - PeriodicMetricReader.builder(exporter) - .setExecutor(Executors.newScheduledThreadPool(0)) - .setInterval(Duration.ofNanos(Long.MAX_VALUE)) - .build() - ) - .build(); - assertBusy(() -> { - GetComposableIndexTemplateAction.Request getReq = new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, "*"); - var templates = client().execute(GetComposableIndexTemplateAction.INSTANCE, getReq).actionGet().indexTemplates(); - assertThat(templates, not(anEmptyMap())); - }); - assumeTrue("Requires otlp_metrics feature flag to be enabled", OTelPlugin.OTLP_METRICS_ENABLED); - } - - private int getHttpPort() { - NodesInfoResponse nodesInfoResponse = client().admin().cluster().prepareNodesInfo().get(); - assertFalse(nodesInfoResponse.hasFailures()); - assertEquals(1, nodesInfoResponse.getNodes().size()); - NodeInfo node = nodesInfoResponse.getNodes().getFirst(); - assertNotNull(node.getInfo(HttpInfo.class)); - TransportAddress publishAddress = node.getInfo(HttpInfo.class).address().publishAddress(); - InetSocketAddress address = publishAddress.address(); - return address.getPort(); - } - - @Override - public void tearDown() throws Exception { - meterProvider.close(); - super.tearDown(); - } - - public void testIngestMetricDataViaMetricExporter() throws Exception { - MetricData jvmMemoryMetricData = createDoubleGauge( - TEST_RESOURCE, - Attributes.empty(), - "jvm.memory.total", - Runtime.getRuntime().totalMemory(), - "By", - Clock.getDefault().now() - ); - - assertThrows("OTLP metrics is not implemented yet", RuntimeException.class, () -> export(List.of(jvmMemoryMetricData))); - } - - private void export(List metrics) throws IOException { - var result = exporter.export(metrics).join(10, TimeUnit.SECONDS); - Throwable failure = result.getFailureThrowable(); - if (failure instanceof FailedExportException.HttpExportException httpExportException) { - throw new RuntimeException(GrpcExporterUtil.getStatusMessage(httpExportException.getResponse().responseBody())); - } else if (failure != null) { - throw new RuntimeException("Failed to export metrics", failure); - } - assertThat(result.isSuccess(), is(true)); - admin().indices().prepareRefresh().execute().actionGet(); - } - - private static MetricData createDoubleGauge( - Resource resource, - Attributes attributes, - String name, - double value, - String unit, - long timeEpochNanos - ) { - return ImmutableMetricData.createDoubleGauge( - resource, - TEST_SCOPE, - name, - "Your description could be here.", - unit, - ImmutableGaugeData.create(List.of(ImmutableDoublePointData.create(timeEpochNanos, timeEpochNanos, attributes, value))) - ); - } -} diff --git a/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java b/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java new file mode 100644 index 0000000000000..72ae74bb3d48d --- /dev/null +++ b/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java @@ -0,0 +1,146 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.action.otlp; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.exporter.internal.FailedExportException; +import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; +import io.opentelemetry.sdk.common.Clock; +import io.opentelemetry.sdk.common.InstrumentationScopeInfo; +import io.opentelemetry.sdk.metrics.SdkMeterProvider; +import io.opentelemetry.sdk.metrics.data.MetricData; +import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableDoublePointData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData; +import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData; +import io.opentelemetry.sdk.resources.Resource; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Before; +import org.junit.ClassRule; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static io.opentelemetry.api.common.AttributeKey.stringKey; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class OTLPMetricsIndexingRestIT extends ESRestTestCase { + + private static final String USER = "test_admin"; + private static final String PASS = "x-pack-test-password"; + private static final Resource TEST_RESOURCE = Resource.create(Attributes.of(stringKey("service.name"), "elasticsearch")); + private static final InstrumentationScopeInfo TEST_SCOPE = InstrumentationScopeInfo.create("io.opentelemetry.example.metrics"); + private OtlpHttpMetricExporter exporter; + private SdkMeterProvider meterProvider; + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .user(USER, PASS, "superuser", false) + .setting("xpack.security.autoconfiguration.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); + return Settings.builder().put(super.restClientSettings()).put(ThreadContext.PREFIX + ".Authorization", token).build(); + } + + @Before + public void beforeTest() throws Exception { + exporter = OtlpHttpMetricExporter.builder() + .setEndpoint(getClusterHosts().getFirst().toURI() + "/_otlp/v1/metrics") + .addHeader("Authorization", basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()))) + .build(); + meterProvider = SdkMeterProvider.builder() + .registerMetricReader( + PeriodicMetricReader.builder(exporter) + .setExecutor(Executors.newScheduledThreadPool(0)) + .setInterval(Duration.ofNanos(Long.MAX_VALUE)) + .build() + ) + .build(); + assertBusy(() -> assertOK(client().performRequest(new Request("GET", "_index_template/metrics-otel@template")))); + boolean otlpEndpointEnabled = false; + try { + otlpEndpointEnabled = RestStatus.isSuccessful( + client().performRequest(new Request("POST", "/_otlp/v1/metrics")).getStatusLine().getStatusCode() + ); + } catch (Exception ignore) {} + assumeTrue("Requires otlp_metrics feature flag to be enabled", otlpEndpointEnabled); + } + + @Override + public void tearDown() throws Exception { + meterProvider.close(); + super.tearDown(); + } + + public void testIngestMetricDataViaMetricExporter() throws Exception { + MetricData jvmMemoryMetricData = createDoubleGauge( + TEST_RESOURCE, + Attributes.empty(), + "jvm.memory.total", + Runtime.getRuntime().totalMemory(), + "By", + Clock.getDefault().now() + ); + + FailedExportException.HttpExportException exception = assertThrows( + FailedExportException.HttpExportException.class, + () -> export(List.of(jvmMemoryMetricData)) + ); + assertThat(exception.getResponse().statusCode(), equalTo(RestStatus.NOT_IMPLEMENTED.getStatus())); + } + + private void export(List metrics) throws Exception { + var result = exporter.export(metrics).join(10, TimeUnit.SECONDS); + Throwable failure = result.getFailureThrowable(); + if (failure instanceof Exception e) { + throw e; + } else if (failure != null) { + throw new RuntimeException("Failed to export metrics", failure); + } + assertThat(result.isSuccess(), is(true)); + assertOK(client().performRequest(new Request("GET", "_refresh/metrics-*"))); + } + + private static MetricData createDoubleGauge( + Resource resource, + Attributes attributes, + String name, + double value, + String unit, + long timeEpochNanos + ) { + return ImmutableMetricData.createDoubleGauge( + resource, + TEST_SCOPE, + name, + "Your description could be here.", + unit, + ImmutableGaugeData.create(List.of(ImmutableDoublePointData.create(timeEpochNanos, timeEpochNanos, attributes, value))) + ); + } +} diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java index 72e1d6006bfa9..c22790a7d5d61 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/OTelPlugin.java @@ -36,8 +36,6 @@ public class OTelPlugin extends Plugin implements ActionPlugin { - public static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled(); - // OTEL_DATA_REGISTRY_ENABLED controls enabling the index template registry. // // This setting will be ignored if the plugin is disabled. @@ -50,6 +48,7 @@ public class OTelPlugin extends Plugin implements ActionPlugin { private static final Logger logger = LogManager.getLogger(OTelPlugin.class); + private static final boolean OTLP_METRICS_ENABLED = new FeatureFlag("otlp_metrics").isEnabled(); private final SetOnce registry = new SetOnce<>(); private final boolean enabled; diff --git a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java index 2b7e83540f73a..722b727fff40f 100644 --- a/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java +++ b/x-pack/plugin/otel-data/src/main/java/org/elasticsearch/xpack/oteldata/otlp/OTLPMetricsTransportAction.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.CompositeIndicesRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.client.internal.Client; @@ -64,7 +65,7 @@ protected void doExecute(Task task, MetricsRequest request, ActionListener Date: Wed, 27 Aug 2025 16:02:41 +0200 Subject: [PATCH 8/8] Use explicit type rater than var for exporter result --- .../elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java b/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java index 72ae74bb3d48d..12b30e743194f 100644 --- a/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java +++ b/x-pack/plugin/otel-data/src/javaRestTest/java/org/elasticsearch/action/otlp/OTLPMetricsIndexingRestIT.java @@ -11,6 +11,7 @@ import io.opentelemetry.exporter.internal.FailedExportException; import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter; import io.opentelemetry.sdk.common.Clock; +import io.opentelemetry.sdk.common.CompletableResultCode; import io.opentelemetry.sdk.common.InstrumentationScopeInfo; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.data.MetricData; @@ -115,7 +116,7 @@ public void testIngestMetricDataViaMetricExporter() throws Exception { } private void export(List metrics) throws Exception { - var result = exporter.export(metrics).join(10, TimeUnit.SECONDS); + CompletableResultCode result = exporter.export(metrics).join(10, TimeUnit.SECONDS); Throwable failure = result.getFailureThrowable(); if (failure instanceof Exception e) { throw e;