feat: use golang echo as reverse proxy (#623)

Author: olevski

Dockerfile.revproxy

@@ -0,0 +1,10 @@
+FROM golang:1.19.5-alpine3.17 as builder
+WORKDIR /src
+COPY go.mod go.sum ./
+COPY cmd/revproxy/ ./
+RUN go build -o /revproxy
+
+FROM alpine:3.17
+USER 1000:1000
+COPY --from=builder /revproxy /revproxy
+ENTRYPOINT [ "/revproxy" ]

app/__init__.py

@@ -214,7 +214,6 @@ def auth():
     if (
         "anon-id" not in request.cookies
         and request.headers.get("X-Requested-With", "") == "XMLHttpRequest"
-        and request.headers["X-Forwarded-Uri"] == "/api/user"
         and "Authorization" not in headers
     ):
         resp = Response(

chartpress.yaml

@@ -9,8 +9,10 @@ charts:
       - .
     images:
       renku-gateway:
-        # Context to send to docker build for use by the Dockerfile
         contextPath: .
-        # Dockerfile path relative to chartpress.yaml
         dockerfilePath: Dockerfile
         valuesPath: image.auth
+      renku-revproxy:
+        contextPath: .
+        dockerfilePath: Dockerfile.revproxy
+        valuesPath: reverseProxy.image

cmd/revproxy/config.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"net/url"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+)
+
+type renkuServicesConfig struct {
+	Notebooks *url.URL `mapstructure:"renku_services_notebooks"`
+	KG        *url.URL `mapstructure:"renku_services_kg"`
+	Webhook   *url.URL `mapstructure:"renku_services_webhook"`
+	Core      *url.URL `mapstructure:"renku_services_core"`
+	Auth      *url.URL `mapstructure:"renku_services_auth"`
+}
+
+type metricsConfig struct {
+	Enabled bool `mapstructure:"metrics_enabled"`
+	Port    int  `mapstructure:"metrics_port"`
+}
+
+type revProxyConfig struct {
+	RenkuBaseURL      *url.URL            `mapstructure:"renku_base_url"`
+	AllowOrigin       []string            `mapstructure:"allow_origin"`
+	ExternalGitlabURL *url.URL            `mapstructure:"external_gitlab_url"`
+	RenkuServices     renkuServicesConfig `mapstructure:",squash"`
+	Metrics           metricsConfig       `mapstructure:",squash"`
+	Port              int
+}
+
+func parseStringAsURL() mapstructure.DecodeHookFuncType {
+	return func(
+		f reflect.Type,
+		t reflect.Type,
+		data interface{},
+	) (interface{}, error) {
+		// Check that the data is string
+		if f.Kind() != reflect.String {
+			return data, nil
+		}
+
+		// Check that the target type is our custom type
+		if t != reflect.TypeOf(url.URL{}) {
+			return data, nil
+		}
+
+		// Return the parsed value
+		dataStr, ok := data.(string)
+		if !ok {
+			return nil, fmt.Errorf("cannot cast URL value to string")
+		}
+		if dataStr == "" {
+			return nil, fmt.Errorf("empty values are not allowed for URLs")
+		}
+		url, err := url.Parse(dataStr)
+		if err != nil {
+			return nil, err
+		}
+		return url, nil
+	}
+}
+
+func getConfig() revProxyConfig {
+	var config revProxyConfig
+	prefix := "revproxy"
+	viper.SetEnvPrefix(prefix)
+	viper.AutomaticEnv()
+	viper.AllowEmptyEnv(false)
+	envKeysMap := &map[string]interface{}{}
+	if err := mapstructure.Decode(config, &envKeysMap); err != nil {
+		log.Fatal(err)
+	}
+	for k := range *envKeysMap {
+		if _, ok := os.LookupEnv(strings.ToUpper(prefix) + "_" + strings.ToUpper(k)); !ok {
+			log.Fatalf("Environment variable %s is not defined\n", strings.ToUpper(prefix)+"_"+strings.ToUpper(k))
+		}
+		if bindErr := viper.BindEnv(k); bindErr != nil {
+			log.Fatal(bindErr)
+		}
+	}
+	err := viper.Unmarshal(&config, viper.DecodeHook(parseStringAsURL()))
+	if err != nil {
+		log.Fatalf("unable to decode config into struct, %v\n", err)
+	}
+	return config
+}
+
+// AddQueryParams makes a copy of the provided URL, adds the query parameters
+// and returns a url with the added parameters. The original URL is left unchanged.
+func AddQueryParams(url *url.URL, params map[string]string) *url.URL {
+	newURL := *url
+	query := newURL.Query()
+	for k, v := range params {
+		query.Add(k, v)
+	}
+	newURL.RawQuery = query.Encode()
+	return &newURL
+}

cmd/revproxy/main.go

@@ -0,0 +1,118 @@
+// Package main contains the definition of all routes, proxying and authentication
+// performed by the reverse proxy that is part of the Renku gateway.
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/labstack/echo/v4/middleware"
+)
+
+func setupServer(config revProxyConfig) *echo.Echo {
+	// Intialize common reverse proxy middlewares
+	fallbackProxy := proxyFromURL(config.RenkuBaseURL)
+	renkuBaseProxyHost := setHost(config.RenkuBaseURL.Host)
+	var gitlabProxy, gitlabProxyHost echo.MiddlewareFunc
+	if config.ExternalGitlabURL != nil {
+		gitlabProxy = proxyFromURL(config.ExternalGitlabURL)
+		gitlabProxyHost = setHost(config.ExternalGitlabURL.Host)
+	} else {
+		gitlabProxy = fallbackProxy
+		gitlabProxyHost = setHost(config.RenkuBaseURL.Host)
+	}
+	notebooksProxy := proxyFromURL(config.RenkuServices.Notebooks)
+	authSvcProxy := proxyFromURL(config.RenkuServices.Auth)
+	coreProxy := proxyFromURL(config.RenkuServices.Core)
+	kgProxy := proxyFromURL(config.RenkuServices.KG)
+	webhookProxy := proxyFromURL(config.RenkuServices.Webhook)
+	logger := middleware.Logger()
+
+	// Initialize common authentication middleware
+	notebooksAuth := authenticate(AddQueryParams(config.RenkuServices.Auth, map[string]string{"auth": "notebook"}), "Renku-Auth-Access-Token", "Renku-Auth-Id-Token", "Renku-Auth-Git-Credentials", "Renku-Auth-Anon-Id", "Renku-Auth-Refresh-Token")
+	renkuAuth := authenticate(AddQueryParams(config.RenkuServices.Auth, map[string]string{"auth": "renku"}), "Authorization", "Renku-user-id", "Renku-user-fullname", "Renku-user-email")
+	gitlabAuth := authenticate(AddQueryParams(config.RenkuServices.Auth, map[string]string{"auth": "gitlab"}), "Authorization")
+	cliGitlabAuth := authenticate(AddQueryParams(config.RenkuServices.Auth, map[string]string{"auth": "cli-gitlab"}), "Authorization")
+
+	// Server instance
+	e := echo.New()
+	e.Pre(middleware.RemoveTrailingSlash())
+	e.Use(middleware.Recover())
+
+	// Routing for Renku services
+	e.Group("/api/auth", logger, authSvcProxy)
+	e.Group("/api/notebooks", logger, notebooksAuth, noCookies, stripPrefix("/api"), notebooksProxy)
+	e.Group("/api/projects/:projectID/graph", logger, gitlabAuth, noCookies, kgProjectsGraphRewrites, webhookProxy)
+	e.Group("/api/datasets", logger, noCookies, regexRewrite("^/api(.*)", "/knowledge-graph$1"), kgProxy)
+	e.Group("/api/kg", logger, gitlabAuth, noCookies, regexRewrite("^/api/kg(.*)", "/knowledge-graph$1"), kgProxy)
+	e.Group("/api/renku", logger, renkuAuth, noCookies, stripPrefix("/api"), coreProxy)
+
+	// Routes that end up proxied to Gitlab
+	if config.ExternalGitlabURL != nil {
+		// Redirect "old" style bundled /gitlab pathing if an external Gitlab is used
+		e.Group("/gitlab", logger, stripPrefix("/gitlab"), gitlabProxyHost, gitlabProxy)
+		e.Group("/api/graphql", logger, gitlabAuth, gitlabProxyHost, gitlabProxy)
+		e.Group("/api/direct", logger, stripPrefix("/api/direct"), gitlabProxyHost, gitlabProxy)
+		e.Group("/api/repos", logger, cliGitlabAuth, noCookies, stripPrefix("/api/repos"), gitlabProxyHost, gitlabProxy)
+		// If nothing is matched in any other more specific /api route then fall back to Gitlab
+		e.Group("/api", logger, gitlabAuth, noCookies, regexRewrite("^/api(.*)", "/api/v4$1"), gitlabProxyHost, gitlabProxy)
+	} else {
+		e.Group("/api/graphql", logger, gitlabAuth, regexRewrite("^(.*)", "/gitlab$1"), gitlabProxyHost, gitlabProxy)
+		e.Group("/api/direct", logger, regexRewrite("^/api/direct(.*)", "/gitlab$1"), gitlabProxyHost, gitlabProxy)
+		e.Group("/api/repos", logger, cliGitlabAuth, noCookies, regexRewrite("^/api/repos(.*)", "/gitlab$1"), gitlabProxyHost, gitlabProxy)
+		// If nothing is matched in any other more specific /api route then fall back to Gitlab
+		e.Group("/api", logger, gitlabAuth, noCookies, regexRewrite("^/api(.*)", "/gitlab/api/v4$1"), gitlabProxyHost, gitlabProxy)
+	}
+
+	// If nothing is matched from any of the routes above then fall back to the UI
+	e.Group("/", logger, renkuBaseProxyHost, fallbackProxy)
+
+	// Reverse proxy specific endpoints
+	rp := e.Group("/revproxy")
+	rp.GET("/health", func(c echo.Context) error {
+		return c.JSON(http.StatusOK, map[string]string{"status": "ok"})
+	})
+
+	return e
+}
+
+func main() {
+	config := getConfig()
+	e := setupServer(config)
+	// Start API server
+	e.Logger.Printf("Starting server with config: %+v", config)
+	go func() {
+		if err := e.Start(fmt.Sprintf(":%d", config.Port)); err != nil && err != http.ErrServerClosed {
+			e.Logger.Fatal(err)
+		}
+	}()
+	// Start metrics server if enabled
+	var metricsServer *echo.Echo
+	if (config.Metrics.Enabled) {
+		metricsServer = getMetricsServer(e, config.Metrics.Port)
+		go func() {
+			if err := metricsServer.Start(fmt.Sprintf(":%d", config.Metrics.Port)); err != nil && err != http.ErrServerClosed {
+				metricsServer.Logger.Fatal(err)
+			}
+		}()
+	}
+	quit := make(chan os.Signal, 1)
+	signal.Notify(quit, os.Interrupt)
+	<-quit  // Wait for interrupt signal from OS
+	// Start shutting down servers
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	if err := e.Shutdown(ctx); err != nil {
+		e.Logger.Fatal(err)
+	}
+	if config.Metrics.Enabled {
+		if err := metricsServer.Shutdown(ctx); err != nil {
+			metricsServer.Logger.Fatal(err)
+		}
+	}
+}