UTF-8 support in metric and label names

benchmarks/src/main/java/io/prometheus/metrics/benchmarks/TextFormatUtilBenchmark.java

@@ -3,6 +3,7 @@
 import io.prometheus.metrics.expositionformats.ExpositionFormatWriter;
 import io.prometheus.metrics.expositionformats.OpenMetricsTextFormatWriter;
 import io.prometheus.metrics.expositionformats.PrometheusTextFormatWriter;
+import io.prometheus.metrics.model.snapshots.EscapingScheme;
 import io.prometheus.metrics.model.snapshots.GaugeSnapshot;
 import io.prometheus.metrics.model.snapshots.GaugeSnapshot.GaugeDataPointSnapshot;
 import io.prometheus.metrics.model.snapshots.Labels;
@@ -69,14 +70,15 @@ public OutputStream openMetricsWriteToByteArray(WriterState writerState) throws
     // avoid growing the array
     ByteArrayOutputStream byteArrayOutputStream = writerState.byteArrayOutputStream;
     byteArrayOutputStream.reset();
-    OPEN_METRICS_TEXT_FORMAT_WRITER.write(byteArrayOutputStream, SNAPSHOTS);
+    OPEN_METRICS_TEXT_FORMAT_WRITER.write(
+        byteArrayOutputStream, SNAPSHOTS, EscapingScheme.NO_ESCAPING);
     return byteArrayOutputStream;
   }
 
   @Benchmark
   public OutputStream openMetricsWriteToNull() throws IOException {
     OutputStream nullOutputStream = NullOutputStream.INSTANCE;
-    OPEN_METRICS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS);
+    OPEN_METRICS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS, EscapingScheme.NO_ESCAPING);
     return nullOutputStream;
   }
 
@@ -85,14 +87,15 @@ public OutputStream prometheusWriteToByteArray(WriterState writerState) throws I
     // avoid growing the array
     ByteArrayOutputStream byteArrayOutputStream = writerState.byteArrayOutputStream;
     byteArrayOutputStream.reset();
-    PROMETHEUS_TEXT_FORMAT_WRITER.write(byteArrayOutputStream, SNAPSHOTS);
+    PROMETHEUS_TEXT_FORMAT_WRITER.write(
+        byteArrayOutputStream, SNAPSHOTS, EscapingScheme.NO_ESCAPING);
     return byteArrayOutputStream;
   }
 
   @Benchmark
   public OutputStream prometheusWriteToNull() throws IOException {
     OutputStream nullOutputStream = NullOutputStream.INSTANCE;
-    PROMETHEUS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS);
+    PROMETHEUS_TEXT_FORMAT_WRITER.write(nullOutputStream, SNAPSHOTS, EscapingScheme.NO_ESCAPING);
     return nullOutputStream;
   }
 

prometheus-metrics-config/src/main/java/io/prometheus/metrics/config/ExporterPushgatewayProperties.java

@@ -7,15 +7,19 @@ public class ExporterPushgatewayProperties {
   private static final String ADDRESS = "address";
   private static final String JOB = "job";
   private static final String SCHEME = "scheme";
+  private static final String ESCAPING_SCHEME = "escapingScheme";
   private static final String PREFIX = "io.prometheus.exporter.pushgateway";
   private final String scheme;
   private final String address;
   private final String job;
+  private final String escapingScheme;
 
-  private ExporterPushgatewayProperties(String address, String job, String scheme) {
+  private ExporterPushgatewayProperties(
+      String address, String job, String scheme, String escapingScheme) {
     this.address = address;
     this.job = job;
     this.scheme = scheme;
+    this.escapingScheme = escapingScheme;
   }
 
   /** Address of the Pushgateway in the form {@code host:port}. Default is {@code localhost:9091} */
@@ -39,6 +43,14 @@ public String getScheme() {
     return scheme;
   }
 
+  /**
+   * Escaping scheme to be used when pushing metric data to the pushgateway. Valid values:
+   * "no-escaping", "values", "underscores", "dots". Default is "no-escaping".
+   */
+  public String getEscapingScheme() {
+    return escapingScheme;
+  }
+
   /**
    * Note that this will remove entries from {@code properties}. This is because we want to know if
    * there are unused properties remaining after all properties have been loaded.
@@ -48,6 +60,8 @@ static ExporterPushgatewayProperties load(Map<Object, Object> properties)
     String address = Util.loadString(PREFIX + "." + ADDRESS, properties);
     String job = Util.loadString(PREFIX + "." + JOB, properties);
     String scheme = Util.loadString(PREFIX + "." + SCHEME, properties);
+    String escapingScheme = Util.loadString(PREFIX + "." + ESCAPING_SCHEME, properties);
+
     if (scheme != null) {
       if (!scheme.equals("http") && !scheme.equals("https")) {
         throw new PrometheusPropertiesException(
@@ -56,6 +70,20 @@ static ExporterPushgatewayProperties load(Map<Object, Object> properties)
                 PREFIX, SCHEME, scheme));
       }
     }
-    return new ExporterPushgatewayProperties(address, job, scheme);
+
+    if (escapingScheme != null) {
+      if (!escapingScheme.equals("no-escaping")
+          && !escapingScheme.equals("values")
+          && !escapingScheme.equals("underscores")
+          && !escapingScheme.equals("dots")) {
+        throw new PrometheusPropertiesException(
+            String.format(
+                "%s.%s: Illegal value. Expecting 'no-escaping', 'values', 'underscores', "
+                    + "or 'dots'. Found: %s",
+                PREFIX, ESCAPING_SCHEME, escapingScheme));
+      }
+    }
+
+    return new ExporterPushgatewayProperties(address, job, scheme, escapingScheme);
   }
 }

prometheus-metrics-config/src/test/java/io/prometheus/metrics/config/PrometheusPropertiesTest.java

@@ -4,6 +4,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Properties;
 import org.junit.jupiter.api.Test;
@@ -37,15 +38,19 @@ public void testBuilder() {
     PrometheusProperties defaults = PrometheusPropertiesLoader.load(new HashMap<>());
     PrometheusProperties.Builder builder = PrometheusProperties.builder();
     builder.defaultMetricsProperties(defaults.getDefaultMetricProperties());
+    builder.metricProperties(
+        Collections.singletonMap(
+            "http_duration_seconds",
+            MetricsProperties.builder().histogramClassicUpperBounds(0.1, 0.2, 0.5, 1.0).build()));
     builder.exemplarProperties(defaults.getExemplarProperties());
     builder.defaultMetricsProperties(defaults.getDefaultMetricProperties());
     builder.exporterFilterProperties(defaults.getExporterFilterProperties());
     builder.exporterHttpServerProperties(defaults.getExporterHttpServerProperties());
     builder.exporterOpenTelemetryProperties(defaults.getExporterOpenTelemetryProperties());
     builder.pushgatewayProperties(defaults.getExporterPushgatewayProperties());
+    builder.exporterProperties(defaults.getExporterProperties());
     PrometheusProperties result = builder.build();
     assertThat(result.getDefaultMetricProperties()).isSameAs(defaults.getDefaultMetricProperties());
-    assertThat(result.getDefaultMetricProperties()).isSameAs(defaults.getDefaultMetricProperties());
     assertThat(result.getExemplarProperties()).isSameAs(defaults.getExemplarProperties());
     assertThat(result.getExporterFilterProperties())
         .isSameAs(defaults.getExporterFilterProperties());
@@ -55,5 +60,11 @@ public void testBuilder() {
         .isSameAs(defaults.getExporterOpenTelemetryProperties());
     assertThat(result.getExporterPushgatewayProperties())
         .isSameAs(defaults.getExporterPushgatewayProperties());
+    assertThat(result.getMetricProperties("http_duration_seconds"))
+        .usingRecursiveComparison()
+        .isEqualTo(
+            MetricsProperties.builder().histogramClassicUpperBounds(0.1, 0.2, 0.5, 1.0).build());
+    assertThat(result.getMetricProperties("unknown_metric")).isNull();
+    assertThat(result.getExporterProperties()).isSameAs(defaults.getExporterProperties());
   }
 }

prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/CounterTest.java

@@ -12,6 +12,7 @@
 import io.prometheus.metrics.expositionformats.internal.PrometheusProtobufWriterImpl;
 import io.prometheus.metrics.expositionformats.internal.ProtobufUtil;
 import io.prometheus.metrics.model.snapshots.CounterSnapshot;
+import io.prometheus.metrics.model.snapshots.EscapingScheme;
 import io.prometheus.metrics.model.snapshots.Exemplar;
 import io.prometheus.metrics.model.snapshots.Label;
 import io.prometheus.metrics.model.snapshots.Labels;
@@ -23,6 +24,8 @@
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 class CounterTest {
 
@@ -104,22 +107,21 @@ public void testLabels() {
     assertThat(getValue(labels, "l", "b")).isCloseTo(3.0, offset(.001));
   }
 
-  @Test
-  public void testTotalStrippedFromName() {
-    for (String name :
-        new String[] {
-          "my_counter_total", "my.counter.total",
-          "my_counter_seconds_total", "my.counter.seconds.total",
-          "my_counter", "my.counter",
-          "my_counter_seconds", "my.counter.seconds"
-        }) {
-      Counter counter = Counter.builder().name(name).unit(Unit.SECONDS).build();
-      Metrics.MetricFamily protobufData =
-          new PrometheusProtobufWriterImpl().convert(counter.collect());
-      assertThat(ProtobufUtil.shortDebugString(protobufData))
-          .isEqualTo(
-              "name: \"my_counter_seconds_total\" type: COUNTER metric { counter { value: 0.0 } }");
-    }
+  @ParameterizedTest
+  @ValueSource(
+      strings = {
+        "my_counter_total",
+        "my_counter_seconds_total",
+        "my_counter",
+        "my_counter_seconds",
+      })
+  public void testTotalStrippedFromName(String name) {
+    Counter counter = Counter.builder().name(name).unit(Unit.SECONDS).build();
+    Metrics.MetricFamily protobufData =
+        new PrometheusProtobufWriterImpl().convert(counter.collect(), EscapingScheme.NO_ESCAPING);
+    assertThat(ProtobufUtil.shortDebugString(protobufData))
+        .isEqualTo(
+            "name: \"my_counter_seconds_total\" type: COUNTER metric { counter { value: 0.0 } }");
   }
 
   @Test

prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/HistogramTest.java

@@ -13,12 +13,7 @@
 import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_4_31_1.Metrics;
 import io.prometheus.metrics.expositionformats.internal.PrometheusProtobufWriterImpl;
 import io.prometheus.metrics.expositionformats.internal.ProtobufUtil;
-import io.prometheus.metrics.model.snapshots.ClassicHistogramBucket;
-import io.prometheus.metrics.model.snapshots.Exemplar;
-import io.prometheus.metrics.model.snapshots.Exemplars;
-import io.prometheus.metrics.model.snapshots.HistogramSnapshot;
-import io.prometheus.metrics.model.snapshots.Labels;
-import io.prometheus.metrics.model.snapshots.MetricSnapshots;
+import io.prometheus.metrics.model.snapshots.*;
 import io.prometheus.metrics.tracer.common.SpanContext;
 import io.prometheus.metrics.tracer.initializer.SpanContextSupplier;
 import java.io.ByteArrayOutputStream;
@@ -91,7 +86,8 @@ private void run() throws NoSuchFieldException, IllegalAccessException {
         }
       }
       Metrics.MetricFamily protobufData =
-          new PrometheusProtobufWriterImpl().convert(histogram.collect());
+          new PrometheusProtobufWriterImpl()
+              .convert(histogram.collect(), EscapingScheme.NO_ESCAPING);
       String expectedWithMetadata =
           "name: \"test\" type: HISTOGRAM metric { histogram { " + expected + " } }";
       assertThat(ProtobufUtil.shortDebugString(protobufData))
@@ -946,13 +942,14 @@ public void testDefaults() throws IOException {
         """;
 
     // protobuf
-    Metrics.MetricFamily protobufData = new PrometheusProtobufWriterImpl().convert(snapshot);
+    Metrics.MetricFamily protobufData =
+        new PrometheusProtobufWriterImpl().convert(snapshot, EscapingScheme.NO_ESCAPING);
     assertThat(ProtobufUtil.shortDebugString(protobufData)).isEqualTo(expectedProtobuf);
 
     // text
     ByteArrayOutputStream out = new ByteArrayOutputStream();
     OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(false, true);
-    writer.write(out, MetricSnapshots.of(snapshot));
+    writer.write(out, MetricSnapshots.of(snapshot), EscapingScheme.NO_ESCAPING);
     assertThat(out).hasToString(expectedTextFormat);
   }
 
@@ -1259,12 +1256,6 @@ public void testIllegalLabelNamePrefix() {
         .isThrownBy(() -> Histogram.builder().name("test").labelNames("__hello"));
   }
 
-  @Test
-  public void testIllegalName() {
-    assertThatExceptionOfType(IllegalArgumentException.class)
-        .isThrownBy(() -> Histogram.builder().name("my_namespace/server.durations"));
-  }
-
   @Test
   public void testNoName() {
     assertThatExceptionOfType(IllegalArgumentException.class)

prometheus-metrics-core/src/test/java/io/prometheus/metrics/core/metrics/InfoTest.java

@@ -7,34 +7,30 @@
 import io.prometheus.metrics.expositionformats.generated.com_google_protobuf_4_31_1.Metrics;
 import io.prometheus.metrics.expositionformats.internal.PrometheusProtobufWriterImpl;
 import io.prometheus.metrics.expositionformats.internal.ProtobufUtil;
+import io.prometheus.metrics.model.snapshots.EscapingScheme;
 import io.prometheus.metrics.model.snapshots.Labels;
 import io.prometheus.metrics.model.snapshots.MetricSnapshots;
 import io.prometheus.metrics.model.snapshots.Unit;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 class InfoTest {
 
-  @Test
-  public void testInfoStrippedFromName() {
-    for (String name :
-        new String[] {
-          "jvm.runtime", "jvm_runtime",
-          "jvm.runtime.info", "jvm_runtime_info"
-        }) {
-      for (String labelName : new String[] {"my.key", "my_key"}) {
-        Info info = Info.builder().name(name).labelNames(labelName).build();
-        info.addLabelValues("value");
-        Metrics.MetricFamily protobufData =
-            new PrometheusProtobufWriterImpl().convert(info.collect());
-        assertThat(ProtobufUtil.shortDebugString(protobufData))
-            .isEqualTo(
-                "name: \"jvm_runtime_info\" type: GAUGE metric { label { name: \"my_key\" value:"
-                    + " \"value\" } gauge { value: 1.0 } }");
-      }
-    }
+  @ParameterizedTest
+  @ValueSource(strings = {"jvm.runtime", "jvm.runtime.info"})
+  public void testInfoStrippedFromName(String name) {
+    Info info = Info.builder().name(name).labelNames("my.key").build();
+    info.addLabelValues("value");
+    Metrics.MetricFamily protobufData =
+        new PrometheusProtobufWriterImpl().convert(info.collect(), EscapingScheme.NO_ESCAPING);
+    assertThat(ProtobufUtil.shortDebugString(protobufData))
+        .isEqualTo(
+            "name: \"jvm.runtime_info\" type: GAUGE metric { label { name: \"my.key\" value:"
+                + " \"value\" } gauge { value: 1.0 } }");
   }
 
   @Test
@@ -126,7 +122,8 @@ public void testConstLabelsDuplicate2() {
   private void assertTextFormat(String expected, Info info) throws IOException {
     OpenMetricsTextFormatWriter writer = new OpenMetricsTextFormatWriter(true, true);
     ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
-    writer.write(outputStream, MetricSnapshots.of(info.collect()));
+    writer.write(
+        outputStream, MetricSnapshots.of(info.collect()), EscapingScheme.UNDERSCORE_ESCAPING);
     String result = outputStream.toString(StandardCharsets.UTF_8.name());
     if (!result.contains(expected)) {
       throw new AssertionError(expected + " is not contained in the following output:\n" + result);

prometheus-metrics-exporter-common/src/main/java/io/prometheus/metrics/exporter/common/PrometheusScrapeHandler.java

@@ -6,6 +6,7 @@
 import io.prometheus.metrics.expositionformats.ExpositionFormats;
 import io.prometheus.metrics.model.registry.MetricNameFilter;
 import io.prometheus.metrics.model.registry.PrometheusRegistry;
+import io.prometheus.metrics.model.snapshots.EscapingScheme;
 import io.prometheus.metrics.model.snapshots.MetricSnapshots;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -54,10 +55,11 @@ public void handleRequest(PrometheusHttpExchange exchange) throws IOException {
     try {
       PrometheusHttpRequest request = exchange.getRequest();
       MetricSnapshots snapshots = scrape(request);
-      if (writeDebugResponse(snapshots, exchange)) {
+      String acceptHeader = request.getHeader("Accept");
+      EscapingScheme escapingScheme = EscapingScheme.fromAcceptHeader(acceptHeader);
+      if (writeDebugResponse(snapshots, escapingScheme, exchange)) {
         return;
       }
-      String acceptHeader = request.getHeader("Accept");
       ExpositionFormatWriter writer = expositionFormats.findWriter(acceptHeader);
       PrometheusHttpResponse response = exchange.getResponse();
       response.setHeader("Content-Type", writer.getContentType());
@@ -66,12 +68,12 @@ public void handleRequest(PrometheusHttpExchange exchange) throws IOException {
         response.setHeader("Content-Encoding", "gzip");
         try (GZIPOutputStream gzipOutputStream =
             new GZIPOutputStream(response.sendHeadersAndGetBody(200, 0))) {
-          writer.write(gzipOutputStream, snapshots);
+          writer.write(gzipOutputStream, snapshots, escapingScheme);
         }
       } else {
         ByteArrayOutputStream responseBuffer =
             new ByteArrayOutputStream(lastResponseSize.get() + 1024);
-        writer.write(responseBuffer, snapshots);
+        writer.write(responseBuffer, snapshots, escapingScheme);
         lastResponseSize.set(responseBuffer.size());
         int contentLength = responseBuffer.size();
         if (contentLength > 0) {
@@ -136,7 +138,8 @@ private MetricSnapshots scrape(PrometheusHttpRequest request) {
     }
   }
 
-  private boolean writeDebugResponse(MetricSnapshots snapshots, PrometheusHttpExchange exchange)
+  private boolean writeDebugResponse(
+      MetricSnapshots snapshots, EscapingScheme escapingScheme, PrometheusHttpExchange exchange)
       throws IOException {
     String debugParam = exchange.getRequest().getParameter("debug");
     PrometheusHttpResponse response = exchange.getResponse();
@@ -148,14 +151,16 @@ private boolean writeDebugResponse(MetricSnapshots snapshots, PrometheusHttpExch
       OutputStream body = response.sendHeadersAndGetBody(responseStatus, 0);
       switch (debugParam) {
         case "openmetrics":
-          expositionFormats.getOpenMetricsTextFormatWriter().write(body, snapshots);
+          expositionFormats.getOpenMetricsTextFormatWriter().write(body, snapshots, escapingScheme);
           break;
         case "text":
-          expositionFormats.getPrometheusTextFormatWriter().write(body, snapshots);
+          expositionFormats.getPrometheusTextFormatWriter().write(body, snapshots, escapingScheme);
           break;
         case "prometheus-protobuf":
           String debugString =
-              expositionFormats.getPrometheusProtobufWriter().toDebugString(snapshots);
+              expositionFormats
+                  .getPrometheusProtobufWriter()
+                  .toDebugString(snapshots, escapingScheme);
           body.write(debugString.getBytes(StandardCharsets.UTF_8));
           break;
         default: