diff --git a/context.go b/context.go
index eb0979f3a61..4b9e8bcbbd6 100644
--- a/context.go
+++ b/context.go
@@ -49,7 +49,7 @@ type Context struct {
 	ancestry        []Module
 	cleanupFuncs    []func()                // invoked at every config unload
 	exitFuncs       []func(context.Context) // invoked at config unload ONLY IF the process is exiting (EXPERIMENTAL)
-	metricsRegistry *prometheus.Registry
+	metricsRegistry *registryGatherer
 }
 
 // NewContext provides a new context derived from the given
@@ -61,7 +61,8 @@ type Context struct {
 // modules which are loaded will be properly unloaded.
 // See standard library context package's documentation.
 func NewContext(ctx Context) (Context, context.CancelFunc) {
-	newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: prometheus.NewPedanticRegistry()}
+	r := prometheus.NewPedanticRegistry()
+	newCtx := Context{moduleInstances: make(map[string][]Module), cfg: ctx.cfg, metricsRegistry: &registryGatherer{registry: r, gatherer: r}}
 	c, cancel := context.WithCancel(ctx.Context)
 	wrappedCancel := func() {
 		cancel()
@@ -103,7 +104,7 @@ func (ctx *Context) FileSystems() FileSystems {
 
 // Returns the active metrics registry for the context
 // EXPERIMENTAL: This API is subject to change.
-func (ctx *Context) GetMetricsRegistry() *prometheus.Registry {
+func (ctx *Context) GetMetricsRegistry() MetricsRegistererGatherer {
 	return ctx.metricsRegistry
 }
 
diff --git a/metrics.go b/metrics.go
index 0ee3853eb85..41154067c7a 100644
--- a/metrics.go
+++ b/metrics.go
@@ -1,9 +1,11 @@
 package caddy
 
 import (
+	"errors"
 	"net/http"
 
 	"github.com/prometheus/client_golang/prometheus"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 
 	"github.com/caddyserver/caddy/v2/internal/metrics"
 )
@@ -82,3 +84,64 @@ func (d *delegator) WriteHeader(code int) {
 func (d *delegator) Unwrap() http.ResponseWriter {
 	return d.ResponseWriter
 }
+
+type MetricsRegistererGatherer interface {
+	prometheus.Registerer
+	prometheus.Gatherer
+}
+type registryGatherer struct {
+	registry prometheus.Registerer
+	gatherer prometheus.Gatherer
+}
+
+// Gather implements prometheus.Gatherer.
+func (r *registryGatherer) Gather() ([]*io_prometheus_client.MetricFamily, error) {
+	return r.gatherer.Gather()
+}
+
+// MustRegister calls `MustRegister` on the backing registry one collector
+// at a time to capture the module at which the call may have panicked. Panics
+// of duplicate registration are ignored.
+func (r *registryGatherer) MustRegister(cs ...prometheus.Collector) {
+	var current prometheus.Collector
+	defer func() {
+		if r := recover(); r != nil {
+			err, ok := r.(error)
+			if !ok {
+				panic(r)
+			}
+			if !errors.Is(err, prometheus.AlreadyRegisteredError{
+				ExistingCollector: current,
+				NewCollector:      current,
+			}) {
+				panic(err)
+			}
+		}
+	}()
+	for _, current = range cs {
+		r.registry.MustRegister(current)
+	}
+}
+
+// Register implements prometheus.Registerer. Errors of duplicate registration
+// are ignored.
+func (r *registryGatherer) Register(c prometheus.Collector) error {
+	if err := r.registry.Register(c); err != nil &&
+		!errors.Is(err, prometheus.AlreadyRegisteredError{
+			ExistingCollector: c,
+			NewCollector:      c,
+		}) {
+		return err
+	}
+	return nil
+}
+
+// Unregister implements prometheus.Registerer.
+func (r *registryGatherer) Unregister(c prometheus.Collector) bool {
+	return r.registry.Unregister(c)
+}
+
+var (
+	_ prometheus.Registerer = (*registryGatherer)(nil)
+	_ prometheus.Gatherer   = (*registryGatherer)(nil)
+)
diff --git a/modules/caddyhttp/reverseproxy/metrics.go b/modules/caddyhttp/reverseproxy/metrics.go
index 2488427304e..7ac78bfba71 100644
--- a/modules/caddyhttp/reverseproxy/metrics.go
+++ b/modules/caddyhttp/reverseproxy/metrics.go
@@ -1,7 +1,6 @@
 package reverseproxy
 
 import (
-	"errors"
 	"runtime/debug"
 	"sync"
 	"time"
@@ -19,7 +18,7 @@ var reverseProxyMetrics = struct {
 	logger           *zap.Logger
 }{}
 
-func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
+func initReverseProxyMetrics(handler *Handler, registry prometheus.Registerer) {
 	const ns, sub = "caddy", "reverse_proxy"
 
 	upstreamsLabels := []string{"upstream"}
@@ -32,17 +31,7 @@ func initReverseProxyMetrics(handler *Handler, registry *prometheus.Registry) {
 		}, upstreamsLabels)
 	})
 
-	// duplicate registration could happen if multiple sites with reverse proxy are configured; so ignore the error because
-	// there's no good way to capture having multiple sites with reverse proxy. If this happens, the metrics will be
-	// registered twice, but the second registration will be ignored.
-	if err := registry.Register(reverseProxyMetrics.upstreamsHealthy); err != nil &&
-		!errors.Is(err, prometheus.AlreadyRegisteredError{
-			ExistingCollector: reverseProxyMetrics.upstreamsHealthy,
-			NewCollector:      reverseProxyMetrics.upstreamsHealthy,
-		}) {
-		panic(err)
-	}
-
+	registry.MustRegister(reverseProxyMetrics.upstreamsHealthy)
 	reverseProxyMetrics.logger = handler.logger.Named("reverse_proxy.metrics")
 }
 
diff --git a/modules/metrics/adminmetrics.go b/modules/metrics/adminmetrics.go
index 1e3a841ddb2..6f990187cca 100644
--- a/modules/metrics/adminmetrics.go
+++ b/modules/metrics/adminmetrics.go
@@ -18,8 +18,6 @@ import (
 	"errors"
 	"net/http"
 
-	"github.com/prometheus/client_golang/prometheus"
-
 	"github.com/caddyserver/caddy/v2"
 )
 
@@ -33,7 +31,7 @@ func init() {
 // See the Metrics module for a configurable endpoint that is usable if the
 // Admin API is disabled.
 type AdminMetrics struct {
-	registry *prometheus.Registry
+	registry caddy.MetricsRegistererGatherer
 
 	metricsHandler http.Handler
 }
diff --git a/modules/metrics/metrics.go b/modules/metrics/metrics.go
index 42b30d88dd7..184f57f9026 100644
--- a/modules/metrics/metrics.go
+++ b/modules/metrics/metrics.go
@@ -15,10 +15,8 @@
 package metrics
 
 import (
-	"errors"
 	"net/http"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
 
@@ -64,9 +62,6 @@ func (l *zapLogger) Println(v ...any) {
 func (m *Metrics) Provision(ctx caddy.Context) error {
 	log := ctx.Logger()
 	registry := ctx.GetMetricsRegistry()
-	if registry == nil {
-		return errors.New("no metrics registry found")
-	}
 	m.metricsHandler = createMetricsHandler(&zapLogger{log}, !m.DisableOpenMetrics, registry)
 	return nil
 }
@@ -112,7 +107,7 @@ var (
 	_ caddyfile.Unmarshaler       = (*Metrics)(nil)
 )
 
-func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry *prometheus.Registry) http.Handler {
+func createMetricsHandler(logger promhttp.Logger, enableOpenMetrics bool, registry caddy.MetricsRegistererGatherer) http.Handler {
 	return promhttp.InstrumentMetricHandler(registry,
 		promhttp.HandlerFor(registry, promhttp.HandlerOpts{
 			// will only log errors if logger is non-nil