From ff9769d41c2399d58c531818b8ff163ac8991ecd Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 13:57:48 -0400 Subject: [PATCH 01/31] Install envoy and configs --- images/chromium-headful/Dockerfile | 18 ++++++ images/chromium-headful/wrapper.sh | 2 + images/chromium-headless/image/Dockerfile | 18 ++++++ images/chromium-headless/image/wrapper.sh | 2 + shared/envoy/bootstrap.yaml | 40 +++++++++++++ shared/envoy/default.yaml | 71 +++++++++++++++++++++++ shared/envoy/init-envoy.sh | 35 +++++++++++ shared/envoy/supervisor-envoy.conf | 9 +++ 8 files changed, 195 insertions(+) create mode 100644 shared/envoy/bootstrap.yaml create mode 100644 shared/envoy/default.yaml create mode 100644 shared/envoy/init-envoy.sh create mode 100644 shared/envoy/supervisor-envoy.conf diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 9a1c9da2..360da896 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -168,12 +168,30 @@ COPY --from=client /src/dist/ /var/www COPY --from=xorg-deps /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so COPY --from=xorg-deps /usr/local/lib/xorg/modules/input/neko_drv.so /usr/lib/xorg/modules/input/neko_drv.so +# Install Envoy proxy (official apt.envoyproxy.io) and add bootstrap configuration +ENV ENVOY_PACKAGE=envoy-1.32 +RUN set -eux; \ + mkdir -p /etc/apt/keyrings; \ + curl -fsSL https://apt.envoyproxy.io/signing.key | gpg --dearmor -o /etc/apt/keyrings/envoy-keyring.gpg; \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/envoy-keyring.gpg] https://apt.envoyproxy.io jammy main" > /etc/apt/sources.list.d/envoy.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends "${ENVOY_PACKAGE}" || (apt-cache policy "${ENVOY_PACKAGE}" envoy && exit 1); \ + apt-mark hold "${ENVOY_PACKAGE}"; \ + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/ +RUN mkdir -p /etc/envoy/templates +COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml +COPY shared/envoy/default.yaml /etc/envoy/default.yaml +COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh +RUN chmod +x /usr/local/bin/init-envoy.sh + COPY images/chromium-headful/image-chromium/ / COPY images/chromium-headful/start-chromium.sh /images/chromium-headful/start-chromium.sh RUN chmod +x /images/chromium-headful/start-chromium.sh COPY images/chromium-headful/wrapper.sh /wrapper.sh COPY images/chromium-headful/supervisord.conf /etc/supervisor/supervisord.conf COPY images/chromium-headful/supervisor/services/ /etc/supervisor/conf.d/services/ +COPY shared/envoy/supervisor-envoy.conf /etc/supervisor/conf.d/services/envoy.conf # copy the kernel-images API binary built in the builder stage COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api diff --git a/images/chromium-headful/wrapper.sh b/images/chromium-headful/wrapper.sh index dae8192a..f42dadea 100755 --- a/images/chromium-headful/wrapper.sh +++ b/images/chromium-headful/wrapper.sh @@ -148,6 +148,8 @@ fi sleep 0.2 done +init-envoy.sh + echo "[wrapper] Starting Xorg via supervisord" supervisorctl -c /etc/supervisor/supervisord.conf start xorg echo "[wrapper] Waiting for Xorg to open display $DISPLAY..." diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index e4bd956b..b763eb8d 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -50,6 +50,23 @@ RUN set -xe; \ software-properties-common \ supervisor; +# Install Envoy proxy (official apt.envoyproxy.io) and add bootstrap configuration +ENV ENVOY_PACKAGE=envoy-1.32 +RUN set -eux; \ + mkdir -p /etc/apt/keyrings; \ + curl -fsSL https://apt.envoyproxy.io/signing.key | gpg --dearmor -o /etc/apt/keyrings/envoy-keyring.gpg; \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/envoy-keyring.gpg] https://apt.envoyproxy.io jammy main" > /etc/apt/sources.list.d/envoy.list; \ + apt-get update; \ + apt-get install -y --no-install-recommends "${ENVOY_PACKAGE}" || (apt-cache policy "${ENVOY_PACKAGE}" envoy && exit 1); \ + apt-mark hold "${ENVOY_PACKAGE}"; \ + apt-get clean -y; \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/ +RUN mkdir -p /etc/envoy/templates +COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml +COPY shared/envoy/default.yaml /etc/envoy/default.yaml +COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh +RUN chmod +x /usr/local/bin/init-envoy.sh + # install chromium and sqlite3 for debugging the cookies file RUN add-apt-repository -y ppa:xtradeb/apps RUN apt update -y && apt install -y chromium sqlite3 @@ -83,6 +100,7 @@ COPY images/chromium-headless/image/wrapper.sh /usr/bin/wrapper.sh # Supervisord configuration COPY images/chromium-headless/image/supervisord.conf /etc/supervisor/supervisord.conf COPY images/chromium-headless/image/supervisor/services/ /etc/supervisor/conf.d/services/ +COPY shared/envoy/supervisor-envoy.conf /etc/supervisor/conf.d/services/envoy.conf # Copy the kernel-images API binary built in the builder stage COPY --from=server-builder /out/kernel-images-api /usr/local/bin/kernel-images-api diff --git a/images/chromium-headless/image/wrapper.sh b/images/chromium-headless/image/wrapper.sh index 0dd79522..33ce7107 100755 --- a/images/chromium-headless/image/wrapper.sh +++ b/images/chromium-headless/image/wrapper.sh @@ -194,6 +194,8 @@ for i in {1..30}; do sleep 0.2 done +init-envoy.sh + echo "[wrapper] Starting system D-Bus daemon via supervisord" supervisorctl -c /etc/supervisor/supervisord.conf start dbus for i in {1..50}; do diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml new file mode 100644 index 00000000..9fd14bde --- /dev/null +++ b/shared/envoy/bootstrap.yaml @@ -0,0 +1,40 @@ +node: + id: "{INSTANCE_NAME}-{METRO_NAME}" + +dynamic_resources: + ads_config: + api_type: GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_server + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 + +static_resources: + clusters: + - name: xds_server + type: STRICT_DNS + connect_timeout: 2s + http2_protocol_options: {} + load_assignment: + cluster_name: xds_server + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: control-plane + port_value: 18000 + +admin: + address: + socket_address: + address: 127.0.0.1 + port_value: 9901 + + diff --git a/shared/envoy/default.yaml b/shared/envoy/default.yaml new file mode 100644 index 00000000..565129a1 --- /dev/null +++ b/shared/envoy/default.yaml @@ -0,0 +1,71 @@ +static_resources: + listeners: + - name: http_explicit_forward_proxy + address: + socket_address: + address: 0.0.0.0 + port_value: 3128 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + stat_prefix: hcm + normalize_path: true + http_filters: + - name: envoy.filters.http.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig + dns_cache_config: + name: local_dns_cache + dns_lookup_family: V4_ONLY + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + route_config: + name: local_route + virtual_hosts: + - name: forward_proxy + domains: ["*"] + routes: + - match: { connect_matcher: {} } + route: + cluster: dynamic_forward_proxy_cluster + upgrade_configs: + - upgrade_type: CONNECT + connect_config: {} + - match: { prefix: "/" } + route: + cluster: dynamic_forward_proxy_cluster + access_log: + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format: "[%START_TIME%] %DOWNSTREAM_REMOTE_ADDRESS% %REQ(:method)% %REQ(:authority)% %REQ(:path)% -> %RESPONSE_CODE% (%BYTES_SENT%b) %DURATION%ms %RESPONSE_FLAGS% %UPSTREAM_TRANSPORT_FAILURE_REASON%\n" + + clusters: + - name: dynamic_forward_proxy_cluster + connect_timeout: 5s + lb_policy: CLUSTER_PROVIDED + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http_protocol_options: {} + upstream_http_protocol_options: + auto_sni: true + auto_san_validation: true + cluster_type: + name: envoy.clusters.dynamic_forward_proxy + typed_config: + "@type": type.googleapis.com/envoy.extensions.clusters.dynamic_forward_proxy.v3.ClusterConfig + dns_cache_config: + name: local_dns_cache + dns_lookup_family: V4_ONLY + +admin: + address: + socket_address: { address: 127.0.0.1, port_value: 9901 } + + diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh new file mode 100644 index 00000000..65507cee --- /dev/null +++ b/shared/envoy/init-envoy.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -o pipefail -o errexit -o nounset + +echo "[envoy-init] Preparing Envoy bootstrap configuration" +mkdir -p /etc/envoy + +render_from_template=false +if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" ]]; then + render_from_template=true +fi + +if $render_from_template; then + inst_esc=$(printf '%s' "$INST_NAME" | sed -e 's/[\/&]/\\&/g') + metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') + sed -e "s|{INSTANCE_NAME}|$inst_esc|g" \ + -e "s|{METRO_NAME}|$metro_esc|g" \ + /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml +else + cp -f /etc/envoy/default.yaml /etc/envoy/bootstrap.yaml +fi + +echo "[envoy-init] Starting Envoy via supervisord" +supervisorctl -c /etc/supervisor/supervisord.conf start envoy +echo "[envoy-init] Waiting for Envoy admin on 127.0.0.1:9901..." +for i in {1..50}; do + if (echo >/dev/tcp/127.0.0.1/9901) >/dev/null 2>&1; then + echo "[envoy-init] Envoy is started" + break + fi + sleep 0.1 + if [[ $i -eq 50 ]]; then + echo "[envoy-init] Failed to start Envoy - admin interface not responding after 5 seconds" + fi +done \ No newline at end of file diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf new file mode 100644 index 00000000..44eb071a --- /dev/null +++ b/shared/envoy/supervisor-envoy.conf @@ -0,0 +1,9 @@ +[program:envoy] +command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-info}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy-1.32 $args' +autostart=false +autorestart=true +startsecs=2 +stdout_logfile=/var/log/supervisord/envoy +redirect_stderr=true + + From 1ca9fb3780915f67684cde17d0c1608101792181 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 14:04:27 -0400 Subject: [PATCH 02/31] Configure chrome to use proxy for tests --- server/e2e/e2e_chromium_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/e2e/e2e_chromium_test.go b/server/e2e/e2e_chromium_test.go index 0b53420a..1233fccc 100644 --- a/server/e2e/e2e_chromium_test.go +++ b/server/e2e/e2e_chromium_test.go @@ -131,7 +131,10 @@ func runChromiumUserDataSavingFlow(t *testing.T, image, containerName string) { } if strings.Contains(image, "headful") { // headless image sets its own flags, so only do this for headful - env["CHROMIUM_FLAGS"] = "--no-sandbox --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-zygote --password-store=basic --no-first-run" + env["CHROMIUM_FLAGS"] = "--no-sandbox --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-zygote --password-store=basic --no-first-run --proxy-server=http://127.0.0.1:3128" + } else { + // headless image respects CHROMIUM_FLAGS env var + env["CHROMIUM_FLAGS"] = "--proxy-server=http://127.0.0.1:3128" } logger.Info("[setup]", "action", "starting container", "image", image, "name", containerName) _, exitCh, err := runContainer(baseCtx, image, containerName, env) From fd5411ed5593c966870d49b6ac7441915578a658 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 14:10:14 -0400 Subject: [PATCH 03/31] Fix default copied location --- images/chromium-headful/Dockerfile | 3 ++- images/chromium-headless/image/Dockerfile | 3 ++- shared/envoy/init-envoy.sh | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 360da896..c37918bd 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -181,7 +181,8 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* /var/cache/apt/ RUN mkdir -p /etc/envoy/templates COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml -COPY shared/envoy/default.yaml /etc/envoy/default.yaml +# Copy default config to bootstrap.yaml so supervisor can start envoy immediately +COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh RUN chmod +x /usr/local/bin/init-envoy.sh diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index b763eb8d..d36248ea 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -63,7 +63,8 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* /var/cache/apt/ RUN mkdir -p /etc/envoy/templates COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml -COPY shared/envoy/default.yaml /etc/envoy/default.yaml +# Copy default config to bootstrap.yaml so supervisor can start envoy immediately +COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh RUN chmod +x /usr/local/bin/init-envoy.sh diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 65507cee..c793723d 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -11,12 +11,14 @@ if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${MET fi if $render_from_template; then + echo "[envoy-init] Rendering template with INST_NAME=${INST_NAME} and METRO_NAME=${METRO_NAME}" inst_esc=$(printf '%s' "$INST_NAME" | sed -e 's/[\/&]/\\&/g') metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') sed -e "s|{INSTANCE_NAME}|$inst_esc|g" \ -e "s|{METRO_NAME}|$metro_esc|g" \ /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml else + echo "[envoy-init] Using default configuration (template vars not provided)" cp -f /etc/envoy/default.yaml /etc/envoy/bootstrap.yaml fi From 8b71927e8c51b7cc7e470b27929b75dc32d1cc3c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 14:26:53 -0400 Subject: [PATCH 04/31] Only test headful --- server/e2e/e2e_chromium_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/e2e/e2e_chromium_test.go b/server/e2e/e2e_chromium_test.go index 1233fccc..c33c179f 100644 --- a/server/e2e/e2e_chromium_test.go +++ b/server/e2e/e2e_chromium_test.go @@ -132,9 +132,6 @@ func runChromiumUserDataSavingFlow(t *testing.T, image, containerName string) { if strings.Contains(image, "headful") { // headless image sets its own flags, so only do this for headful env["CHROMIUM_FLAGS"] = "--no-sandbox --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-zygote --password-store=basic --no-first-run --proxy-server=http://127.0.0.1:3128" - } else { - // headless image respects CHROMIUM_FLAGS env var - env["CHROMIUM_FLAGS"] = "--proxy-server=http://127.0.0.1:3128" } logger.Info("[setup]", "action", "starting container", "image", image, "name", containerName) _, exitCh, err := runContainer(baseCtx, image, containerName, env) From 23e916280b2b43ca02b8146d6c928755147963d6 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 14:32:00 -0400 Subject: [PATCH 05/31] delete unneeded copy --- shared/envoy/init-envoy.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index c793723d..2ccd4318 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -18,8 +18,7 @@ if $render_from_template; then -e "s|{METRO_NAME}|$metro_esc|g" \ /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml else - echo "[envoy-init] Using default configuration (template vars not provided)" - cp -f /etc/envoy/default.yaml /etc/envoy/bootstrap.yaml + echo "[envoy-init] Using default configuration (template vars INST_NAME and METRO_NAME not provided)" fi echo "[envoy-init] Starting Envoy via supervisord" From cd6eaf96b7ad9ebbd3a8420b5d08223c15f4422a Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 14:41:43 -0400 Subject: [PATCH 06/31] Fix executable name --- shared/envoy/supervisor-envoy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index 44eb071a..d98bf7c4 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-info}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy-1.32 $args' +command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-info}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy $args' autostart=false autorestart=true startsecs=2 From fde7806c26e4b36362bf6bc806a53de93ea94493 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Fri, 19 Sep 2025 15:03:07 -0400 Subject: [PATCH 07/31] Configure tls + jwt auth --- shared/envoy/bootstrap.yaml | 10 +++++++++- shared/envoy/init-envoy.sh | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 9fd14bde..b344323b 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -8,6 +8,9 @@ dynamic_resources: grpc_services: - envoy_grpc: cluster_name: xds_server + initial_metadata: + - key: "authorization" + value: "Bearer {XDS_JWT}" lds_config: ads: {} resource_api_version: V3 @@ -21,6 +24,11 @@ static_resources: type: STRICT_DNS connect_timeout: 2s http2_protocol_options: {} + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: {XDS_SERVER} load_assignment: cluster_name: xds_server endpoints: @@ -28,7 +36,7 @@ static_resources: - endpoint: address: socket_address: - address: control-plane + address: {XDS_SERVER} port_value: 18000 admin: diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 2ccd4318..4768ebd0 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -6,19 +6,23 @@ echo "[envoy-init] Preparing Envoy bootstrap configuration" mkdir -p /etc/envoy render_from_template=false -if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" ]]; then +if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" && -n "${XDS_SERVER:-}" && -n "${XDS_JWT:-}" ]]; then render_from_template=true fi if $render_from_template; then - echo "[envoy-init] Rendering template with INST_NAME=${INST_NAME} and METRO_NAME=${METRO_NAME}" + echo "[envoy-init] Rendering template with INST_NAME=${INST_NAME}, METRO_NAME=${METRO_NAME}, XDS_SERVER=${XDS_SERVER}, XDS_JWT=***" inst_esc=$(printf '%s' "$INST_NAME" | sed -e 's/[\/&]/\\&/g') metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') + xds_esc=$(printf '%s' "$XDS_SERVER" | sed -e 's/[\/&]/\\&/g') + jwt_esc=$(printf '%s' "$XDS_JWT" | sed -e 's/[\/&]/\\&/g') sed -e "s|{INSTANCE_NAME}|$inst_esc|g" \ -e "s|{METRO_NAME}|$metro_esc|g" \ + -e "s|{XDS_SERVER}|$xds_esc|g" \ + -e "s|{XDS_JWT}|$jwt_esc|g" \ /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml else - echo "[envoy-init] Using default configuration (template vars INST_NAME and METRO_NAME not provided)" + echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=${XDS_JWT:+***}${XDS_JWT:-unset})" fi echo "[envoy-init] Starting Envoy via supervisord" From 4caaa98155a9dcc3d4f7fcb9f520baa12db25e6f Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 22 Sep 2025 13:39:46 -0400 Subject: [PATCH 08/31] Fix port --- shared/envoy/bootstrap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index b344323b..f39ec68b 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -37,7 +37,7 @@ static_resources: address: socket_address: address: {XDS_SERVER} - port_value: 18000 + port_value: 443 admin: address: From 2bdad136051672e0320f8882790c0e3e35147041 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 22 Sep 2025 13:59:27 -0400 Subject: [PATCH 09/31] Add docs --- README.md | 11 +++++++++++ shared/envoy/bootstrap.yaml | 21 ++++++++++++++++++--- shared/envoy/default.yaml | 33 ++++++++++++++++++++++++++++++--- 3 files changed, 59 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index afc3f236..b353917f 100644 --- a/README.md +++ b/README.md @@ -165,6 +165,17 @@ curl http://localhost:10001/recording/download --output recording.mp4 Note: the recording file is encoded into a H.264/MPEG-4 AVC video file. [QuickTime has known issues with playback](https://discussions.apple.com/thread/254851789?sortBy=rank) so please make sure to use a compatible media player! +## Proxy configuration + +[Envoy](https://www.envoyproxy.io/) is installed with the browser images, which allows for configuration of a forward proxy chain for egress browser traffic. This proxy is part of the Kernel platform, configured by the following environment variables: + +**INST_NAME**: Instance name in the platform of this browser, used to identify this browser with the platform +**METRO_NAME**: Dataplane server name in the platform of this browser, used to identify this browser with the platform +**XDS_SERVER**: The xDS server hostname, where envoy can discover configuration +**XDS_JWT**: A token used to authenticate this browser to the xDS server + +Envoy is running alongside of the browser. The browser may be configured to proxy through envoy using the [--proxy-server flag](https://www.chromium.org/developers/design-documents/network-settings/). The default configuration directly egresses traffic to the internet. When configured to receive dynamic configuration, the xDS server can control the egress traffic flows of the browser, for example through a forward proxy chain. + ## Documentation This repo powers our managed [browser infrastructure](https://onkernel.com/docs). diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index f39ec68b..3e178373 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -1,16 +1,27 @@ +# Envoy bootstrap configuration for xDS-managed proxy +# This config connects to a control plane for dynamic configuration management +# Requires: INST_NAME, METRO_NAME, XDS_SERVER, XDS_JWT environment variables + +# Node identity sent to xDS server for configuration targeting, authenticated by JWT node: id: "{INSTANCE_NAME}-{METRO_NAME}" +# Dynamic configuration via xDS protocol dynamic_resources: + # Aggregated Discovery Service - single gRPC stream for all config types ads_config: api_type: GRPC transport_api_version: V3 grpc_services: - envoy_grpc: + # Reference to xDS server cluster below cluster_name: xds_server + # Send JWT authentication for all xDS requests initial_metadata: - key: "authorization" value: "Bearer {XDS_JWT}" + + # Listener Discovery Service and Cluster Discovery Service use ADS lds_config: ads: {} resource_api_version: V3 @@ -18,15 +29,20 @@ dynamic_resources: ads: {} resource_api_version: V3 +# Static configuration (always present) static_resources: clusters: + # xDS server: control plane for configuration - name: xds_server + # Resolve hostname via DNS, for DNS lookup type: STRICT_DNS connect_timeout: 2s http2_protocol_options: {} + # TLS configuration for secure xDS connection transport_socket: name: envoy.transport_sockets.tls typed_config: + # Uses TLS to verify xDS server, and SNI hostname for TLS handshake "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext sni: {XDS_SERVER} load_assignment: @@ -39,10 +55,9 @@ static_resources: address: {XDS_SERVER} port_value: 443 +# Envoy admin interface for debugging admin: address: socket_address: address: 127.0.0.1 - port_value: 9901 - - + port_value: 9901 \ No newline at end of file diff --git a/shared/envoy/default.yaml b/shared/envoy/default.yaml index 565129a1..de6026ea 100644 --- a/shared/envoy/default.yaml +++ b/shared/envoy/default.yaml @@ -1,24 +1,38 @@ +# HTTP forward proxy, direct egress to internet static_resources: listeners: - name: http_explicit_forward_proxy address: + # e.g. on chromium, set --proxy-server=http://127.0.0.1:3128 socket_address: address: 0.0.0.0 port_value: 3128 filter_chains: + # One filter chain for HTTP/1.1 proxy traffic - filters: + # HTTP Connection Manager filter: + # this is handling the connection between the client and the proxy, + # which is an HTTP connection. - name: envoy.filters.network.http_connection_manager typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager stat_prefix: hcm normalize_path: true + # Enable forward proxy behavior http_filters: + # Dynamic Forward Proxy filter: resolves upstreams on-the-fly and caches DNS + # Browsers send CONNECT to this proxy to tunnel HTTPS upstreams + # e.g. CONNECT target.example.com + # Envoy establishes a TCP tunnel to the target. - name: envoy.filters.http.dynamic_forward_proxy typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.dynamic_forward_proxy.v3.FilterConfig dns_cache_config: name: local_dns_cache dns_lookup_family: V4_ONLY + # Router filter: actually routes/tunnels the request once target is known + # See what happens here down in route_config, slightly different + # for HTTPS vs HTTP upstream requests. - name: envoy.filters.http.router typed_config: "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router @@ -28,22 +42,35 @@ static_resources: - name: forward_proxy domains: ["*"] routes: + # === HTTPS upstream traffic (CONNECT tunnel) === + # Handle CONNECT method for HTTPS tunneling (creates TCP tunnel) + # 'connect_matcher' is a special matcher that matches CONNECT requests + # "Note that CONNECT support is currently considered alpha in Envoy." + # https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto#envoy-v3-api-field-config-route-v3-routematch-connect-matcher - match: { connect_matcher: {} } route: + # Use single DFP cluster; CONNECT handled via connect_config cluster: dynamic_forward_proxy_cluster + # This tells Envoy to upgrade the connection to a TCP tunnel + # which we only do after getting the CONNECT request. upgrade_configs: - upgrade_type: CONNECT connect_config: {} + # === HTTP upstream traffic (absolute-form proxy) === + # The client didn't send CONNECT, because it's an HTTP request. - match: { prefix: "/" } route: + # Same cluster for HTTP proxying cluster: dynamic_forward_proxy_cluster access_log: + # Access log sink: print one line per request to stdout - name: envoy.access_loggers.stdout typed_config: "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog log_format: text_format: "[%START_TIME%] %DOWNSTREAM_REMOTE_ADDRESS% %REQ(:method)% %REQ(:authority)% %REQ(:path)% -> %RESPONSE_CODE% (%BYTES_SENT%b) %DURATION%ms %RESPONSE_FLAGS% %UPSTREAM_TRANSPORT_FAILURE_REASON%\n" + # Connection pooling / load balancing to target(s) clusters: - name: dynamic_forward_proxy_cluster connect_timeout: 5s @@ -64,8 +91,8 @@ static_resources: name: local_dns_cache dns_lookup_family: V4_ONLY +# Admin interface for debugging and monitoring admin: address: - socket_address: { address: 127.0.0.1, port_value: 9901 } - - + # Admin interface (metrics, config dump, clusters, listeners). Not exposed publicly. + socket_address: { address: 127.0.0.1, port_value: 9901 } \ No newline at end of file From 91c83fd55789c753554da8358f22a3092d7b5d69 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 22 Sep 2025 14:07:36 -0400 Subject: [PATCH 10/31] Fix logging jwt --- shared/envoy/init-envoy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 4768ebd0..9c58e759 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -22,7 +22,7 @@ if $render_from_template; then -e "s|{XDS_JWT}|$jwt_esc|g" \ /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml else - echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=${XDS_JWT:+***}${XDS_JWT:-unset})" + echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=${XDS_JWT:+set}${XDS_JWT:-unset})" fi echo "[envoy-init] Starting Envoy via supervisord" From 159ef5d1eb008fa46dc09a297450d643cfc690ba Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 22 Sep 2025 14:18:11 -0400 Subject: [PATCH 11/31] Fix template render --- shared/envoy/bootstrap.yaml | 2 +- shared/envoy/init-envoy.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 3e178373..b11833b4 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -4,7 +4,7 @@ # Node identity sent to xDS server for configuration targeting, authenticated by JWT node: - id: "{INSTANCE_NAME}-{METRO_NAME}" + id: "{INST_NAME}-{METRO_NAME}" # Dynamic configuration via xDS protocol dynamic_resources: diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 9c58e759..b460f64a 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -16,13 +16,13 @@ if $render_from_template; then metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') xds_esc=$(printf '%s' "$XDS_SERVER" | sed -e 's/[\/&]/\\&/g') jwt_esc=$(printf '%s' "$XDS_JWT" | sed -e 's/[\/&]/\\&/g') - sed -e "s|{INSTANCE_NAME}|$inst_esc|g" \ + sed -e "s|{INST_NAME}|$inst_esc|g" \ -e "s|{METRO_NAME}|$metro_esc|g" \ -e "s|{XDS_SERVER}|$xds_esc|g" \ -e "s|{XDS_JWT}|$jwt_esc|g" \ /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml else - echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=${XDS_JWT:+set}${XDS_JWT:-unset})" + echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT)" fi echo "[envoy-init] Starting Envoy via supervisord" From e63734a3ff82314c9fa3dcfff531a24cc8d07b2b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Mon, 22 Sep 2025 14:30:42 -0400 Subject: [PATCH 12/31] make list --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b353917f..3be78a70 100644 --- a/README.md +++ b/README.md @@ -169,10 +169,10 @@ Note: the recording file is encoded into a H.264/MPEG-4 AVC video file. [QuickTi [Envoy](https://www.envoyproxy.io/) is installed with the browser images, which allows for configuration of a forward proxy chain for egress browser traffic. This proxy is part of the Kernel platform, configured by the following environment variables: -**INST_NAME**: Instance name in the platform of this browser, used to identify this browser with the platform -**METRO_NAME**: Dataplane server name in the platform of this browser, used to identify this browser with the platform -**XDS_SERVER**: The xDS server hostname, where envoy can discover configuration -**XDS_JWT**: A token used to authenticate this browser to the xDS server +- **INST_NAME**: Instance name in the platform of this browser, used to identify this browser with the platform +- **METRO_NAME**: Dataplane server name in the platform of this browser, used to identify this browser with the platform +- **XDS_SERVER**: The xDS server hostname, where envoy can discover configuration +- **XDS_JWT**: A token used to authenticate this browser to the xDS server Envoy is running alongside of the browser. The browser may be configured to proxy through envoy using the [--proxy-server flag](https://www.chromium.org/developers/design-documents/network-settings/). The default configuration directly egresses traffic to the internet. When configured to receive dynamic configuration, the xDS server can control the egress traffic flows of the browser, for example through a forward proxy chain. From 864fdc3149fb0bec402906711e26d7bd70d4d08b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 12:57:23 -0400 Subject: [PATCH 13/31] Envoy log level warning --- shared/envoy/supervisor-envoy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index d98bf7c4..d7f79ff9 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-info}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy $args' +command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-warning}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy $args' autostart=false autorestart=true startsecs=2 From aa1050e31a004f7bfea0772eca34f1e2b1ab2bd8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 14:13:03 -0400 Subject: [PATCH 14/31] Fix indentation --- shared/envoy/bootstrap.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index b11833b4..89535d54 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -16,10 +16,10 @@ dynamic_resources: - envoy_grpc: # Reference to xDS server cluster below cluster_name: xds_server - # Send JWT authentication for all xDS requests - initial_metadata: - - key: "authorization" - value: "Bearer {XDS_JWT}" + # Send JWT authentication for all xDS requests + initial_metadata: + - key: "authorization" + value: "Bearer {XDS_JWT}" # Listener Discovery Service and Cluster Discovery Service use ADS lds_config: From 3361f83861e0cf1e08bdb9464ec5f7f91748769c Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 14:39:17 -0400 Subject: [PATCH 15/31] Fix bootstrap config --- shared/envoy/bootstrap.yaml | 2 +- shared/envoy/supervisor-envoy.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 89535d54..8e339eb8 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -5,6 +5,7 @@ # Node identity sent to xDS server for configuration targeting, authenticated by JWT node: id: "{INST_NAME}-{METRO_NAME}" + cluster: "{INST_NAME}-{METRO_NAME}" # Dynamic configuration via xDS protocol dynamic_resources: @@ -37,7 +38,6 @@ static_resources: # Resolve hostname via DNS, for DNS lookup type: STRICT_DNS connect_timeout: 2s - http2_protocol_options: {} # TLS configuration for secure xDS connection transport_socket: name: envoy.transport_sockets.tls diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index d7f79ff9..82e93a15 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=/bin/bash -lc 'set -e; args="-c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-warning}"; [ -n "${ENVOY_NODE_ID:-}" ] && args="$args --service-node=${ENVOY_NODE_ID}"; [ -n "${ENVOY_CLUSTER:-}" ] && args="$args --service-cluster=${ENVOY_CLUSTER}"; exec envoy $args' +command=envoy -c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-warning} autostart=false autorestart=true startsecs=2 From 4184afb02f07296f3f4565621d1f592ead5a65a8 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 14:48:40 -0400 Subject: [PATCH 16/31] Simplify log level --- shared/envoy/supervisor-envoy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index 82e93a15..85f3befb 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=envoy -c /etc/envoy/bootstrap.yaml --log-level ${ENVOY_LOG_LEVEL:-warning} +command=envoy -c /etc/envoy/bootstrap.yaml --log-level warning autostart=false autorestart=true startsecs=2 From 80834d58e6f456fdd7f43ce029504b474e9a07f5 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 14:53:54 -0400 Subject: [PATCH 17/31] Fix cluster name --- shared/envoy/bootstrap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 8e339eb8..d959473a 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -5,7 +5,7 @@ # Node identity sent to xDS server for configuration targeting, authenticated by JWT node: id: "{INST_NAME}-{METRO_NAME}" - cluster: "{INST_NAME}-{METRO_NAME}" + cluster: "kernel" # Dynamic configuration via xDS protocol dynamic_resources: From 0a6dd7f04853da8160cab5c66800d4abf03ac676 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Tue, 23 Sep 2025 15:24:35 -0400 Subject: [PATCH 18/31] Configure authority header --- shared/envoy/bootstrap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index d959473a..0f2e902a 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -17,6 +17,7 @@ dynamic_resources: - envoy_grpc: # Reference to xDS server cluster below cluster_name: xds_server + authority: "{XDS_SERVER}" # Send JWT authentication for all xDS requests initial_metadata: - key: "authorization" From 61b051ab634ca040a3ea2a6acf45d0800829e8c2 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 08:56:28 -0400 Subject: [PATCH 19/31] fix indentation --- shared/envoy/bootstrap.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 0f2e902a..60080e8d 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -17,7 +17,7 @@ dynamic_resources: - envoy_grpc: # Reference to xDS server cluster below cluster_name: xds_server - authority: "{XDS_SERVER}" + authority: "{XDS_SERVER}" # Send JWT authentication for all xDS requests initial_metadata: - key: "authorization" From 7a225e22216795ae0059455b254280f01e9d9b89 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 16:05:30 -0400 Subject: [PATCH 20/31] Configuration working for both proxied and direct --- images/chromium-headful/.gitignore | 1 + images/chromium-headful/run-docker.sh | 21 +++++++++++++++++++++ images/chromium-headful/run-unikernel.sh | 14 ++++++++++++++ shared/envoy/bootstrap.yaml | 6 ++++-- shared/envoy/init-envoy.sh | 24 ++++++++++++++++++++++++ shared/envoy/supervisor-envoy.conf | 2 +- 6 files changed, 65 insertions(+), 3 deletions(-) diff --git a/images/chromium-headful/.gitignore b/images/chromium-headful/.gitignore index 47a435ce..c9b3f4f0 100644 --- a/images/chromium-headful/.gitignore +++ b/images/chromium-headful/.gitignore @@ -3,3 +3,4 @@ recording/ .tmp/ .rootfs/ initrd +temp.sh diff --git a/images/chromium-headful/run-docker.sh b/images/chromium-headful/run-docker.sh index 62361096..50a9e2fe 100755 --- a/images/chromium-headful/run-docker.sh +++ b/images/chromium-headful/run-docker.sh @@ -18,6 +18,9 @@ CHROMIUM_FLAGS_DEFAULT="--user-data-dir=/home/kernel/user-data --disable-dev-shm if [[ "$RUN_AS_ROOT" == "true" ]]; then CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --no-sandbox --no-zygote" fi +if [[ -n "${XDS_JWT:-}" ]]; then + CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128 --ignore-certificate-errors" +fi CHROMIUM_FLAGS="${CHROMIUM_FLAGS:-$CHROMIUM_FLAGS_DEFAULT}" rm -rf .tmp/chromium mkdir -p .tmp/chromium @@ -43,6 +46,24 @@ RUN_ARGS=( --mount type=bind,src="$FLAGS_FILE",dst=/chromium/flags,ro ) +# Add XDS environment variables if provided +if [[ -n "${INST_NAME:-}" ]]; then + RUN_ARGS+=( -e "INST_NAME=$INST_NAME" ) +fi +if [[ -n "${METRO_NAME:-}" ]]; then + RUN_ARGS+=( -e "METRO_NAME=$METRO_NAME" ) +fi +if [[ -n "${XDS_SERVER:-}" ]]; then + RUN_ARGS+=( -e "XDS_SERVER=$XDS_SERVER" ) +fi +if [[ -n "${XDS_JWT:-}" ]]; then + RUN_ARGS+=( -e "XDS_JWT=$XDS_JWT" ) + RUN_ARGS+=( -p 9901:9901 ) + RUN_ARGS+=( -p 3128:3128 ) + # TODO: import bright data and self-signed client proxy cert to chrome instead of ignore + CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128 --ignore-certificate-errors" +fi + # WebRTC port mapping if [[ "${ENABLE_WEBRTC:-}" == "true" ]]; then echo "Running container with WebRTC" diff --git a/images/chromium-headful/run-unikernel.sh b/images/chromium-headful/run-unikernel.sh index 9dc9cfe5..5c2567ac 100755 --- a/images/chromium-headful/run-unikernel.sh +++ b/images/chromium-headful/run-unikernel.sh @@ -54,6 +54,20 @@ deploy_args=( -n "$NAME" ) +# Add XDS environment variables if provided +if [[ -n "${INST_NAME:-}" ]]; then + deploy_args+=(-e "INST_NAME=$INST_NAME") +fi +if [[ -n "${METRO_NAME:-}" ]]; then + deploy_args+=(-e "METRO_NAME=$METRO_NAME") +fi +if [[ -n "${XDS_SERVER:-}" ]]; then + deploy_args+=(-e "XDS_SERVER=$XDS_SERVER") +fi +if [[ -n "${XDS_JWT:-}" ]]; then + deploy_args+=(-e "XDS_JWT=$XDS_JWT") +fi + if [[ "${ENABLE_WEBRTC:-}" == "true" ]]; then echo "Deploying with WebRTC enabled" kraft cloud inst create --start \ diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 60080e8d..8111188b 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -39,6 +39,8 @@ static_resources: # Resolve hostname via DNS, for DNS lookup type: STRICT_DNS connect_timeout: 2s + http2_protocol_options: {} + dns_lookup_family: V4_ONLY # TLS configuration for secure xDS connection transport_socket: name: envoy.transport_sockets.tls @@ -60,5 +62,5 @@ static_resources: admin: address: socket_address: - address: 127.0.0.1 - port_value: 9901 \ No newline at end of file + address: 0.0.0.0 + port_value: 9901 diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index b460f64a..f44734ee 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -5,6 +5,30 @@ set -o pipefail -o errexit -o nounset echo "[envoy-init] Preparing Envoy bootstrap configuration" mkdir -p /etc/envoy +# Generate self-signed certificates for TLS forward proxy +echo "[envoy-init] Generating self-signed certificates for TLS forward proxy" +mkdir -p /etc/envoy/certs + +if [[ ! -f /etc/envoy/certs/proxy.crt || ! -f /etc/envoy/certs/proxy.key ]]; then + echo "[envoy-init] Creating new self-signed certificate" + openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ + -keyout /etc/envoy/certs/proxy.key \ + -out /etc/envoy/certs/proxy.crt \ + -subj "/C=US/ST=CA/O=Kernel/CN=localhost" \ + -addext "subjectAltName = DNS:localhost,IP:127.0.0.1" \ + 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] Certificate generated successfully" + + # Add certificate to system trust store for Chrome/Chromium + echo "[envoy-init] Adding certificate to system trust store" + mkdir -p /usr/local/share/ca-certificates + cp /etc/envoy/certs/proxy.crt /usr/local/share/ca-certificates/kernel-envoy-proxy.crt + update-ca-certificates 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] Certificate added to system trust store" +else + echo "[envoy-init] Certificates already exist, skipping generation" +fi + render_from_template=false if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" && -n "${XDS_SERVER:-}" && -n "${XDS_JWT:-}" ]]; then render_from_template=true diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index 85f3befb..248990e2 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=envoy -c /etc/envoy/bootstrap.yaml --log-level warning +command=envoy -c /etc/envoy/bootstrap.yaml --log-level info autostart=false autorestart=true startsecs=2 From 4c429766587dd619acbe0cb71e37f743c47c5bda Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 16:57:25 -0400 Subject: [PATCH 21/31] install brightdata certs --- images/chromium-headful/Dockerfile | 13 +++++-- images/chromium-headful/run-docker.sh | 2 +- shared/envoy/init-envoy.sh | 50 +++++++++++++++++++++++---- 3 files changed, 56 insertions(+), 9 deletions(-) diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index c37918bd..9afe9e0c 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -116,7 +116,7 @@ RUN set -eux; \ install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe; \ rm -rf /tmp/ffmpeg* -# runtime + # runtime ENV USERNAME=root RUN set -eux; \ apt-get update; \ @@ -126,7 +126,8 @@ RUN set -eux; \ libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx7 \ gstreamer1.0-plugins-base gstreamer1.0-plugins-good \ gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly \ - gstreamer1.0-pulseaudio gstreamer1.0-omx; \ + gstreamer1.0-pulseaudio gstreamer1.0-omx \ + libnss3-tools; \ # # install libxcvt0 (not available in debian:bullseye) ARCH=$(dpkg --print-architecture); \ @@ -186,6 +187,14 @@ COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh RUN chmod +x /usr/local/bin/init-envoy.sh +# Download and extract BrightData proxy certificate +RUN set -eux; \ + mkdir -p /etc/envoy/brightdata; \ + curl -fsSL https://brightdata.com/static/brightdata_proxy_ca.zip -o /tmp/brightdata_proxy_ca.zip; \ + unzip -j /tmp/brightdata_proxy_ca.zip '*/*.crt' -d /etc/envoy/brightdata/ || true; \ + rm /tmp/brightdata_proxy_ca.zip; \ + echo "BrightData certificates extracted to /etc/envoy/brightdata/" + COPY images/chromium-headful/image-chromium/ / COPY images/chromium-headful/start-chromium.sh /images/chromium-headful/start-chromium.sh RUN chmod +x /images/chromium-headful/start-chromium.sh diff --git a/images/chromium-headful/run-docker.sh b/images/chromium-headful/run-docker.sh index 50a9e2fe..521578fe 100755 --- a/images/chromium-headful/run-docker.sh +++ b/images/chromium-headful/run-docker.sh @@ -19,7 +19,7 @@ if [[ "$RUN_AS_ROOT" == "true" ]]; then CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --no-sandbox --no-zygote" fi if [[ -n "${XDS_JWT:-}" ]]; then - CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128 --ignore-certificate-errors" + CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128" fi CHROMIUM_FLAGS="${CHROMIUM_FLAGS:-$CHROMIUM_FLAGS_DEFAULT}" rm -rf .tmp/chromium diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index f44734ee..564c62cc 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -20,15 +20,53 @@ if [[ ! -f /etc/envoy/certs/proxy.crt || ! -f /etc/envoy/certs/proxy.key ]]; the echo "[envoy-init] Certificate generated successfully" # Add certificate to system trust store for Chrome/Chromium - echo "[envoy-init] Adding certificate to system trust store" - mkdir -p /usr/local/share/ca-certificates - cp /etc/envoy/certs/proxy.crt /usr/local/share/ca-certificates/kernel-envoy-proxy.crt - update-ca-certificates 2>&1 | sed 's/^/[envoy-init] /' - echo "[envoy-init] Certificate added to system trust store" + echo "[envoy-init] Adding certificate to system trust store" + cp /etc/envoy/certs/proxy.crt /usr/local/share/ca-certificates/kernel-envoy-proxy.crt + cp /etc/envoy/certs/proxy.crt /kernel-envoy-proxy.crt + update-ca-certificates 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] Certificate added to system trust store" + if [[ $RUN_AS_ROOT == "true" ]]; then + mkdir -p /root/.pki/nssdb + certutil -d /root/.pki/nssdb -N --empty-password 2>/dev/null || true + certutil -d /root/.pki/nssdb -A -t "C,," -n "Kernel Envoy Proxy" -i /etc/envoy/certs/proxy.crt + echo "[envoy-init] Certificate added to nssdb as root" + else + mkdir -p /home/kernel/.pki/nssdb + certutil -d /home/kernel/.pki/nssdb -N --empty-password 2>/dev/null || true + certutil -d /home/kernel/.pki/nssdb -A -t "C,," -n "Kernel Envoy Proxy" -i /etc/envoy/certs/proxy.crt + chown -R kernel:kernel /home/kernel/.pki + echo "[envoy-init] Certificate added to nssdb as kernel" + fi + echo "[envoy-init] Certificate added to nssdb" else echo "[envoy-init] Certificates already exist, skipping generation" fi +# Install BrightData certificates if they exist +if [[ -d /etc/envoy/brightdata ]] && [[ -n "$(ls -A /etc/envoy/brightdata/*.crt 2>/dev/null)" ]]; then + echo "[envoy-init] Installing BrightData certificates" + for cert in /etc/envoy/brightdata/*.crt; do + cert_name=$(basename "$cert" .crt) + echo "[envoy-init] Processing BrightData certificate: $cert_name" + + # Add to system trust store + cp "$cert" "/usr/local/share/ca-certificates/brightdata-${cert_name}.crt" + + # Add to NSS database + if [[ $RUN_AS_ROOT == "true" ]]; then + certutil -d /root/.pki/nssdb -A -t "C,," -n "BrightData $cert_name" -i "$cert" 2>&1 | sed 's/^/[envoy-init] /' + else + certutil -d /home/kernel/.pki/nssdb -A -t "C,," -n "BrightData $cert_name" -i "$cert" 2>&1 | sed 's/^/[envoy-init] /' + fi + done + + # Update system certificates + update-ca-certificates 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] BrightData certificates installed" +else + echo "[envoy-init] No BrightData certificates found in /etc/envoy/brightdata" +fi + render_from_template=false if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" && -n "${XDS_SERVER:-}" && -n "${XDS_JWT:-}" ]]; then render_from_template=true @@ -61,4 +99,4 @@ for i in {1..50}; do if [[ $i -eq 50 ]]; then echo "[envoy-init] Failed to start Envoy - admin interface not responding after 5 seconds" fi -done \ No newline at end of file +done From a5a80fbfb529c0bb25f79e762a18eddce5fa93bb Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 17:30:38 -0400 Subject: [PATCH 22/31] Add certificates in headless image --- images/chromium-headful/Dockerfile | 25 ++++--------------- images/chromium-headless/image/Dockerfile | 21 +++++++--------- shared/envoy/init-envoy.sh | 2 ++ shared/envoy/install-proxy.sh | 29 +++++++++++++++++++++++ 4 files changed, 44 insertions(+), 33 deletions(-) create mode 100644 shared/envoy/install-proxy.sh diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 9afe9e0c..c831a0d4 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -169,32 +169,17 @@ COPY --from=client /src/dist/ /var/www COPY --from=xorg-deps /usr/local/lib/xorg/modules/drivers/dummy_drv.so /usr/lib/xorg/modules/drivers/dummy_drv.so COPY --from=xorg-deps /usr/local/lib/xorg/modules/input/neko_drv.so /usr/lib/xorg/modules/input/neko_drv.so -# Install Envoy proxy (official apt.envoyproxy.io) and add bootstrap configuration -ENV ENVOY_PACKAGE=envoy-1.32 -RUN set -eux; \ - mkdir -p /etc/apt/keyrings; \ - curl -fsSL https://apt.envoyproxy.io/signing.key | gpg --dearmor -o /etc/apt/keyrings/envoy-keyring.gpg; \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/envoy-keyring.gpg] https://apt.envoyproxy.io jammy main" > /etc/apt/sources.list.d/envoy.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends "${ENVOY_PACKAGE}" || (apt-cache policy "${ENVOY_PACKAGE}" envoy && exit 1); \ - apt-mark hold "${ENVOY_PACKAGE}"; \ - apt-get clean -y; \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/ -RUN mkdir -p /etc/envoy/templates +# Install Envoy proxy and BrightData certificates +COPY shared/envoy/install-proxy.sh /usr/local/bin/install-proxy.sh +RUN chmod +x /usr/local/bin/install-proxy.sh && /usr/local/bin/install-proxy.sh && rm /usr/local/bin/install-proxy.sh + +# Copy Envoy configuration files COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml # Copy default config to bootstrap.yaml so supervisor can start envoy immediately COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml COPY shared/envoy/init-envoy.sh /usr/local/bin/init-envoy.sh RUN chmod +x /usr/local/bin/init-envoy.sh -# Download and extract BrightData proxy certificate -RUN set -eux; \ - mkdir -p /etc/envoy/brightdata; \ - curl -fsSL https://brightdata.com/static/brightdata_proxy_ca.zip -o /tmp/brightdata_proxy_ca.zip; \ - unzip -j /tmp/brightdata_proxy_ca.zip '*/*.crt' -d /etc/envoy/brightdata/ || true; \ - rm /tmp/brightdata_proxy_ca.zip; \ - echo "BrightData certificates extracted to /etc/envoy/brightdata/" - COPY images/chromium-headful/image-chromium/ / COPY images/chromium-headful/start-chromium.sh /images/chromium-headful/start-chromium.sh RUN chmod +x /images/chromium-headful/start-chromium.sh diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index d36248ea..a49b9c7e 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -48,20 +48,15 @@ RUN set -xe; \ xvfb \ x11-utils \ software-properties-common \ - supervisor; + supervisor \ + libnss3-tools \ + unzip; -# Install Envoy proxy (official apt.envoyproxy.io) and add bootstrap configuration -ENV ENVOY_PACKAGE=envoy-1.32 -RUN set -eux; \ - mkdir -p /etc/apt/keyrings; \ - curl -fsSL https://apt.envoyproxy.io/signing.key | gpg --dearmor -o /etc/apt/keyrings/envoy-keyring.gpg; \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/envoy-keyring.gpg] https://apt.envoyproxy.io jammy main" > /etc/apt/sources.list.d/envoy.list; \ - apt-get update; \ - apt-get install -y --no-install-recommends "${ENVOY_PACKAGE}" || (apt-cache policy "${ENVOY_PACKAGE}" envoy && exit 1); \ - apt-mark hold "${ENVOY_PACKAGE}"; \ - apt-get clean -y; \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/ -RUN mkdir -p /etc/envoy/templates +# Install Envoy proxy and BrightData certificates +COPY shared/envoy/install-proxy.sh /usr/local/bin/install-proxy.sh +RUN chmod +x /usr/local/bin/install-proxy.sh && /usr/local/bin/install-proxy.sh && rm /usr/local/bin/install-proxy.sh + +# Copy Envoy configuration files COPY shared/envoy/bootstrap.yaml /etc/envoy/templates/bootstrap.yaml # Copy default config to bootstrap.yaml so supervisor can start envoy immediately COPY shared/envoy/default.yaml /etc/envoy/bootstrap.yaml diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 564c62cc..487f95f1 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -55,8 +55,10 @@ if [[ -d /etc/envoy/brightdata ]] && [[ -n "$(ls -A /etc/envoy/brightdata/*.crt # Add to NSS database if [[ $RUN_AS_ROOT == "true" ]]; then certutil -d /root/.pki/nssdb -A -t "C,," -n "BrightData $cert_name" -i "$cert" 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] Certificate added to nssdb as root" else certutil -d /home/kernel/.pki/nssdb -A -t "C,," -n "BrightData $cert_name" -i "$cert" 2>&1 | sed 's/^/[envoy-init] /' + echo "[envoy-init] Certificate added to nssdb as kernel" fi done diff --git a/shared/envoy/install-proxy.sh b/shared/envoy/install-proxy.sh new file mode 100644 index 00000000..4375b85a --- /dev/null +++ b/shared/envoy/install-proxy.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -eux + +# Install Envoy proxy (official apt.envoyproxy.io) +ENVOY_PACKAGE="${ENVOY_PACKAGE:-envoy-1.32}" + +echo "Installing Envoy proxy package: ${ENVOY_PACKAGE}" +mkdir -p /etc/apt/keyrings +curl -fsSL https://apt.envoyproxy.io/signing.key | gpg --dearmor -o /etc/apt/keyrings/envoy-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/envoy-keyring.gpg] https://apt.envoyproxy.io jammy main" > /etc/apt/sources.list.d/envoy.list +apt-get update +apt-get install -y --no-install-recommends "${ENVOY_PACKAGE}" || (apt-cache policy "${ENVOY_PACKAGE}" envoy && exit 1) +apt-mark hold "${ENVOY_PACKAGE}" +apt-get clean -y +rm -rf /var/lib/apt/lists/* /var/cache/apt/ + +# Create directory structure for Envoy configuration +mkdir -p /etc/envoy/templates + +# Download and extract BrightData proxy certificate +echo "Downloading and extracting BrightData certificates" +mkdir -p /etc/envoy/brightdata +curl -fsSL https://brightdata.com/static/brightdata_proxy_ca.zip -o /tmp/brightdata_proxy_ca.zip +unzip -j /tmp/brightdata_proxy_ca.zip '*/*.crt' -d /etc/envoy/brightdata/ || true +rm /tmp/brightdata_proxy_ca.zip +echo "BrightData certificates extracted to /etc/envoy/brightdata/" + +# List extracted certificates for verification +ls -la /etc/envoy/brightdata/ From dee45a2cf6aeb8ee3b22fae952715fa5e0ceffd3 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 17:33:38 -0400 Subject: [PATCH 23/31] Fixes from code review --- images/chromium-headful/run-docker.sh | 2 -- shared/envoy/init-envoy.sh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/images/chromium-headful/run-docker.sh b/images/chromium-headful/run-docker.sh index 521578fe..1aecc18f 100755 --- a/images/chromium-headful/run-docker.sh +++ b/images/chromium-headful/run-docker.sh @@ -60,8 +60,6 @@ if [[ -n "${XDS_JWT:-}" ]]; then RUN_ARGS+=( -e "XDS_JWT=$XDS_JWT" ) RUN_ARGS+=( -p 9901:9901 ) RUN_ARGS+=( -p 3128:3128 ) - # TODO: import bright data and self-signed client proxy cert to chrome instead of ignore - CHROMIUM_FLAGS_DEFAULT="$CHROMIUM_FLAGS_DEFAULT --proxy-server=https://127.0.0.1:3128 --ignore-certificate-errors" fi # WebRTC port mapping diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index 487f95f1..abf06c67 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -25,7 +25,7 @@ if [[ ! -f /etc/envoy/certs/proxy.crt || ! -f /etc/envoy/certs/proxy.key ]]; the cp /etc/envoy/certs/proxy.crt /kernel-envoy-proxy.crt update-ca-certificates 2>&1 | sed 's/^/[envoy-init] /' echo "[envoy-init] Certificate added to system trust store" - if [[ $RUN_AS_ROOT == "true" ]]; then +if [[ "${RUN_AS_ROOT:-}" == "true" ]]; then mkdir -p /root/.pki/nssdb certutil -d /root/.pki/nssdb -N --empty-password 2>/dev/null || true certutil -d /root/.pki/nssdb -A -t "C,," -n "Kernel Envoy Proxy" -i /etc/envoy/certs/proxy.crt @@ -53,7 +53,7 @@ if [[ -d /etc/envoy/brightdata ]] && [[ -n "$(ls -A /etc/envoy/brightdata/*.crt cp "$cert" "/usr/local/share/ca-certificates/brightdata-${cert_name}.crt" # Add to NSS database - if [[ $RUN_AS_ROOT == "true" ]]; then + if [[ "${RUN_AS_ROOT:-}" == "true" ]]; then certutil -d /root/.pki/nssdb -A -t "C,," -n "BrightData $cert_name" -i "$cert" 2>&1 | sed 's/^/[envoy-init] /' echo "[envoy-init] Certificate added to nssdb as root" else From 38ddb1ee5b94ced293a0ca2b6e2bf77b5b718ee3 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 17:38:05 -0400 Subject: [PATCH 24/31] Don't start with envoy if not set up --- shared/envoy/init-envoy.sh | 44 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index abf06c67..bc54b05e 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -2,6 +2,20 @@ set -o pipefail -o errexit -o nounset +# Check for required environment variables, to see if envoy is enabled +if [[ -z "${INST_NAME:-}" || -z "${METRO_NAME:-}" || -z "${XDS_SERVER:-}" || -z "${XDS_JWT:-}" ]]; then + echo "[envoy-init] Required environment variables not set. Skipping Envoy initialization." + echo "[envoy-init] Required: INST_NAME, METRO_NAME, XDS_SERVER, XDS_JWT" + echo "[envoy-init] Current values: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=***" + exit 0 +fi + +# Also check for template file +if [[ ! -f /etc/envoy/templates/bootstrap.yaml ]]; then + echo "[envoy-init] Template file /etc/envoy/templates/bootstrap.yaml not found. Skipping Envoy initialization." + exit 0 +fi + echo "[envoy-init] Preparing Envoy bootstrap configuration" mkdir -p /etc/envoy @@ -69,25 +83,17 @@ else echo "[envoy-init] No BrightData certificates found in /etc/envoy/brightdata" fi -render_from_template=false -if [[ -f /etc/envoy/templates/bootstrap.yaml && -n "${INST_NAME:-}" && -n "${METRO_NAME:-}" && -n "${XDS_SERVER:-}" && -n "${XDS_JWT:-}" ]]; then - render_from_template=true -fi - -if $render_from_template; then - echo "[envoy-init] Rendering template with INST_NAME=${INST_NAME}, METRO_NAME=${METRO_NAME}, XDS_SERVER=${XDS_SERVER}, XDS_JWT=***" - inst_esc=$(printf '%s' "$INST_NAME" | sed -e 's/[\/&]/\\&/g') - metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') - xds_esc=$(printf '%s' "$XDS_SERVER" | sed -e 's/[\/&]/\\&/g') - jwt_esc=$(printf '%s' "$XDS_JWT" | sed -e 's/[\/&]/\\&/g') - sed -e "s|{INST_NAME}|$inst_esc|g" \ - -e "s|{METRO_NAME}|$metro_esc|g" \ - -e "s|{XDS_SERVER}|$xds_esc|g" \ - -e "s|{XDS_JWT}|$jwt_esc|g" \ - /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml -else - echo "[envoy-init] Using default configuration (template vars not provided: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT)" -fi +# Render template with provided environment variables +echo "[envoy-init] Rendering template with INST_NAME=${INST_NAME}, METRO_NAME=${METRO_NAME}, XDS_SERVER=${XDS_SERVER}, XDS_JWT=***" +inst_esc=$(printf '%s' "$INST_NAME" | sed -e 's/[\/&]/\\&/g') +metro_esc=$(printf '%s' "$METRO_NAME" | sed -e 's/[\/&]/\\&/g') +xds_esc=$(printf '%s' "$XDS_SERVER" | sed -e 's/[\/&]/\\&/g') +jwt_esc=$(printf '%s' "$XDS_JWT" | sed -e 's/[\/&]/\\&/g') +sed -e "s|{INST_NAME}|$inst_esc|g" \ + -e "s|{METRO_NAME}|$metro_esc|g" \ + -e "s|{XDS_SERVER}|$xds_esc|g" \ + -e "s|{XDS_JWT}|$jwt_esc|g" \ + /etc/envoy/templates/bootstrap.yaml > /etc/envoy/bootstrap.yaml echo "[envoy-init] Starting Envoy via supervisord" supervisorctl -c /etc/supervisor/supervisord.conf start envoy From c64c38703391dc73326920881636a5c02c75e89f Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 24 Sep 2025 17:41:59 -0400 Subject: [PATCH 25/31] Update server/e2e/e2e_chromium_test.go --- server/e2e/e2e_chromium_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/e2e/e2e_chromium_test.go b/server/e2e/e2e_chromium_test.go index c33c179f..0b53420a 100644 --- a/server/e2e/e2e_chromium_test.go +++ b/server/e2e/e2e_chromium_test.go @@ -131,7 +131,7 @@ func runChromiumUserDataSavingFlow(t *testing.T, image, containerName string) { } if strings.Contains(image, "headful") { // headless image sets its own flags, so only do this for headful - env["CHROMIUM_FLAGS"] = "--no-sandbox --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-zygote --password-store=basic --no-first-run --proxy-server=http://127.0.0.1:3128" + env["CHROMIUM_FLAGS"] = "--no-sandbox --disable-dev-shm-usage --disable-gpu --start-maximized --disable-software-rasterizer --remote-allow-origins=* --no-zygote --password-store=basic --no-first-run" } logger.Info("[setup]", "action", "starting container", "image", image, "name", containerName) _, exitCh, err := runContainer(baseCtx, image, containerName, env) From 2ab855d406217477dfe4c8d6f5cd9a6a9a42feb0 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 10:24:38 -0400 Subject: [PATCH 26/31] Set log level to warn --- shared/envoy/supervisor-envoy.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/envoy/supervisor-envoy.conf b/shared/envoy/supervisor-envoy.conf index 248990e2..808d59f2 100644 --- a/shared/envoy/supervisor-envoy.conf +++ b/shared/envoy/supervisor-envoy.conf @@ -1,5 +1,5 @@ [program:envoy] -command=envoy -c /etc/envoy/bootstrap.yaml --log-level info +command=envoy -c /etc/envoy/bootstrap.yaml --log-level warn autostart=false autorestart=true startsecs=2 From 941be663240a66c91c869c9c6d88eba69fb83efc Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 10:37:52 -0400 Subject: [PATCH 27/31] Disable admin interface --- shared/envoy/bootstrap.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/shared/envoy/bootstrap.yaml b/shared/envoy/bootstrap.yaml index 8111188b..7b40262b 100644 --- a/shared/envoy/bootstrap.yaml +++ b/shared/envoy/bootstrap.yaml @@ -58,9 +58,9 @@ static_resources: address: {XDS_SERVER} port_value: 443 -# Envoy admin interface for debugging -admin: - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 +# Enable Envoy admin interface for debugging locally +# admin: +# address: +# socket_address: +# address: 0.0.0.0 +# port_value: 9901 From d7782cd3a1335e56b57ead1cd40b89ecdb37be58 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 10:54:56 -0400 Subject: [PATCH 28/31] Less logs when doesn't start --- shared/envoy/init-envoy.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index bc54b05e..d8967eff 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -5,8 +5,6 @@ set -o pipefail -o errexit -o nounset # Check for required environment variables, to see if envoy is enabled if [[ -z "${INST_NAME:-}" || -z "${METRO_NAME:-}" || -z "${XDS_SERVER:-}" || -z "${XDS_JWT:-}" ]]; then echo "[envoy-init] Required environment variables not set. Skipping Envoy initialization." - echo "[envoy-init] Required: INST_NAME, METRO_NAME, XDS_SERVER, XDS_JWT" - echo "[envoy-init] Current values: INST_NAME=${INST_NAME:-unset}, METRO_NAME=${METRO_NAME:-unset}, XDS_SERVER=${XDS_SERVER:-unset}, XDS_JWT=***" exit 0 fi From 0eba8932f81d4e15584203f099aab4d951421371 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 11:04:22 -0400 Subject: [PATCH 29/31] Update images/chromium-headful/Dockerfile --- images/chromium-headful/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index c831a0d4..6b7400ad 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -116,7 +116,7 @@ RUN set -eux; \ install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe; \ rm -rf /tmp/ffmpeg* - # runtime +# runtime ENV USERNAME=root RUN set -eux; \ apt-get update; \ From 01d7303e72e9fef05f7054a0aaae47a0e8ffd64b Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 11:06:53 -0400 Subject: [PATCH 30/31] Update shared/envoy/init-envoy.sh --- shared/envoy/init-envoy.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/shared/envoy/init-envoy.sh b/shared/envoy/init-envoy.sh index d8967eff..581c38df 100644 --- a/shared/envoy/init-envoy.sh +++ b/shared/envoy/init-envoy.sh @@ -95,14 +95,3 @@ sed -e "s|{INST_NAME}|$inst_esc|g" \ echo "[envoy-init] Starting Envoy via supervisord" supervisorctl -c /etc/supervisor/supervisord.conf start envoy -echo "[envoy-init] Waiting for Envoy admin on 127.0.0.1:9901..." -for i in {1..50}; do - if (echo >/dev/tcp/127.0.0.1/9901) >/dev/null 2>&1; then - echo "[envoy-init] Envoy is started" - break - fi - sleep 0.1 - if [[ $i -eq 50 ]]; then - echo "[envoy-init] Failed to start Envoy - admin interface not responding after 5 seconds" - fi -done From bd89af1ef5fc23fdb07a72652bbeef5161aa0a04 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 25 Sep 2025 15:12:20 -0400 Subject: [PATCH 31/31] Log when we connected to the port --- images/chromium-headful/wrapper.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/images/chromium-headful/wrapper.sh b/images/chromium-headful/wrapper.sh index f42dadea..82b1569f 100755 --- a/images/chromium-headful/wrapper.sh +++ b/images/chromium-headful/wrapper.sh @@ -195,6 +195,7 @@ supervisorctl -c /etc/supervisor/supervisord.conf start chromium echo "[wrapper] Waiting for Chromium remote debugging on 127.0.0.1:$INTERNAL_PORT..." for i in {1..100}; do if nc -z 127.0.0.1 "$INTERNAL_PORT" 2>/dev/null; then + echo "connected to chrome debugging port." break fi sleep 0.2