ci: networking experiments with railway & fly

Dockerfile.fly

@@ -0,0 +1,82 @@
+# Dockerfile.fly (based on Dockerfile.jupyter)
+FROM --platform=linux/amd64 python:3.12.7-slim
+
+# Install system dependencies, IPFS, and build tools
+RUN apt-get update && apt-get install -y \
+    git \
+    curl \
+    wget \
+    build-essential \
+    python3-dev \
+    python3-venv \
+    libffi-dev \
+    libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+ARG KUBO_VERSION=0.33.2
+
+# Install IPFS with architecture detection
+RUN arch=$(uname -m) && \
+    if [ "$arch" = "aarch64" ]; then \
+    IPFS_ARCH="arm64"; \
+    else \
+    IPFS_ARCH="amd64"; \
+    fi && \
+    wget "https://dist.ipfs.tech/kubo/v${KUBO_VERSION}/kubo_v${KUBO_VERSION}_linux-${IPFS_ARCH}.tar.gz" && \
+    tar -xvzf "kubo_v${KUBO_VERSION}_linux-${IPFS_ARCH}.tar.gz" && \
+    cd kubo && \
+    bash install.sh && \
+    cd .. && \
+    rm -rf "kubo_v${KUBO_VERSION}_linux-${IPFS_ARCH}.tar.gz"
+
+# UV installation
+RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH"
+
+# Create data directory structure for our single volume
+RUN mkdir -p /data/ipfs /data/venv /data/notebooks /opt/requirements
+
+# Create symbolic links to the expected paths
+RUN ln -s /data/ipfs /root/.ipfs && \
+    ln -s /data/venv /opt/venv && \
+    ln -s /data/notebooks /notebooks
+
+# Create a directory for UV virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN uv venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Install base packages using UV
+COPY requirements.txt .
+RUN uv pip install --no-cache -r requirements.txt
+
+# Expose ports for Jupyter and IPFS
+EXPOSE 8888
+EXPOSE 4001
+EXPOSE 5001
+EXPOSE 8080
+
+# Copy and set up scripts
+COPY scripts/start-fly.sh /scripts/start-fly.sh
+COPY scripts/install_packages.sh /scripts/install_packages.sh
+RUN chmod +x /scripts/start-fly.sh /scripts/install_packages.sh
+
+# Install Node.js and npm
+RUN apt-get update && apt-get install -y nodejs npm
+
+# Create a directory for our Node.js echo server
+WORKDIR /echo-server
+
+# Copy the echo server file into the container
+COPY echo-server.js .
+
+# Initialize a package.json and install the ws package
+RUN npm init -y && npm install ws
+
+# Set the working directory back to /notebooks
+WORKDIR /notebooks
+
+# Use the fly-specific start script
+CMD ["/scripts/start-fly.sh"] 

Dockerfile.jupyter

@@ -63,9 +63,13 @@ RUN uv pip install --no-cache -r requirements.txt
 # Create requirements directory
 RUN mkdir -p /opt/requirements
 
+
 # Initialize IPFS with default settings and enable AutoTLS
-RUN ipfs init && \
-    ipfs config --json AutoTLS '{"Enabled": true, "AutoWSS": true}'
+RUN ipfs init
+
+# # Initialize IPFS with default settings and enable AutoTLS
+# RUN ipfs init && \
+#     ipfs config --json AutoTLS '{"Enabled": true, "AutoWSS": true}'
 
 # Set up working directory
 WORKDIR /notebooks
@@ -88,4 +92,19 @@ COPY scripts/start.sh /scripts/start.sh
 COPY scripts/install_packages.sh /scripts/install_packages.sh
 RUN chmod +x /scripts/start.sh /scripts/install_packages.sh
 
+
+# Install Node.js and npm (using Debian packages available in slim images)
+RUN apt-get update && apt-get install -y nodejs npm
+
+# Create a directory for our Node.js echo server
+WORKDIR /echo-server
+
+# Copy the echo server file into the container
+COPY echo-server.js .
+
+COPY cert.pem key.pem /echo-server/
+
+# Initialize a package.json and install the ws package
+RUN npm init -y && npm install ws
+
 CMD ["/scripts/start.sh"]

FLY_DEPLOYMENT.md

@@ -0,0 +1,108 @@
+# Deploying to Fly.io
+
+This guide explains how to deploy your Jupyter + IPFS environment to Fly.io.
+
+## Prerequisites
+
+1. Install the Fly CLI:
+   ```
+   curl -L https://fly.io/install.sh | sh
+   ```
+   Or use Homebrew on macOS:
+   ```
+   brew install flyctl
+   ```
+
+2. Sign up and authenticate:
+   ```
+   fly auth signup
+   # or if you already have an account
+   fly auth login
+   ```
+
+## Deployment Steps
+
+1. **Create your app first**:
+   ```
+   fly apps create zarr-jupyter
+   ```
+   This reserves your app name in fly.io.
+
+2. **Copy the deployment files to your project**:
+   Make sure these files are set up correctly:
+   - `fly.toml` - Configuration for Fly.io deployment
+   - `Dockerfile.fly` - Docker image for Fly.io
+   - `scripts/start-fly.sh` - Startup script for Fly.io
+   - `scripts/install_packages.sh` - Package installer script
+
+3. **Create the required volumes** (must be done after app creation):
+   ```
+   fly volumes create zarr_ipfs_data --size 10 --region iad
+   fly volumes create zarr_venv_data --size 5 --region iad
+   fly volumes create zarr_notebooks_data --size 10 --region iad
+   ```
+
+   **Important**: Replace `iad` with your chosen region. This must match the `primary_region` in your `fly.toml` file.
+
+4. **Deploy your application**:
+   ```
+   fly deploy --dockerfile Dockerfile.fly
+   ```
+
+5. **Set a secure Jupyter token** (if not already in your fly.toml):
+   ```
+   fly secrets set JUPYTER_TOKEN=your_secure_token
+   ```
+
+## Accessing Your Application
+
+After deployment, you can access your Jupyter environment:
+
+```
+fly open
+```
+
+This will open your browser to the Jupyter Lab interface. Use the JUPYTER_TOKEN you set to log in.
+
+## Monitoring and Management
+
+- View logs:
+  ```
+  fly logs
+  ```
+
+- SSH into the running container:
+  ```
+  fly ssh console
+  ```
+
+- Stop the application:
+  ```
+  fly apps stop
+  ```
+
+- Start the application:
+  ```
+  fly apps start
+  ```
+
+## Troubleshooting
+
+### App Not Found When Creating Volumes
+If you see an error like:
+```
+❯ fly volumes create zarr_ipfs_data --size 10
+Error: failed to list volumes: app not found
+```
+Make sure you've created the app first with `fly apps create zarr-jupyter`.
+
+### Configuration Version Error
+If you see an error about configuration versions, ensure your `fly.toml` is properly formatted for the current version. The example in this repository is compatible with the latest fly.io platform.
+
+## Important Notes
+
+1. The IPFS node is configured to run within your Fly.io application. Ensure your application has enough resources by adjusting the `cpus` and `memory_mb` settings in your `fly.toml` file.
+
+2. The persistent volumes (`zarr_ipfs_data`, `zarr_venv_data`, and `zarr_notebooks_data`) ensure your data persists across deployments.
+
+3. For production use, consider setting a more secure JUPYTER_TOKEN via the secrets management. 

cert.pem

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIFCTCCAvGgAwIBAgIUfl129aAAwRzaufNYm5w/Fmma2WQwDQYJKoZIhvcNAQEL
+BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI1MDMxNjAwMzcwNloXDTI2MDMx
+NjAwMzcwNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF
+AAOCAg8AMIICCgKCAgEAvjm3fAsLuxtQDylmgHelqhEqs0IBdtjwiJ/AkE31khLx
+osQuzXM/dDY1RmuZkS10gLaI0H227/lanloC+EMMuQ1OcEiD+C03CdGliehCD9Oq
+68iFj8DnMB8FyRfYshurChqqThDv6p+S83zLCYcFpMmw4iOQMnYUIcJ4Id7TS1Al
+LM6x6okpjXq4H8qWv8zrw9rSLPagpLynFg4E8F/Qet+8VRVOQ5F1FYgmlO8pfs+1
+hFs9O9L7LCKD33zsHeNrHjv56v2NQGVpNgkQKyC0q+PTotD/QUj37Ehu+UeR8bI9
+EiKbFMHAidLtjChXnqIQkTwWMFgt+Aeq1/5HtVdS5/kYuNygCV5iNZDyH9SMj6ig
+xqaUjrTrfC9vw1pQcKHcTFmmBJlBEnbLCsvt5wl1YBrQ9yVAKAiTwYWyXkz0MAyU
+w3yn6+rCxGNY4r+2hAX4OEkKodCe6qY1YyQfa9YwRG2ZUodwvtOsF31+MK9y2x5p
+ijlU+12EshvRZszAUKOB0l0UeekCc+5h1OlMRbhkr27tOgPQkmTH7eYY6u7NSfpM
+xLQS2N+vgB0hzdujTfXf43XODpSm+uSl1mpmF/LbXZTz6FjU55yvaqAOX15ahIby
+iQN9rt4FKAm1WVf844p1yqQuAz0PBVdpsk6o5HO8EnRycCTmrbYPvxX4NyXv6J0C
+AwEAAaNTMFEwHQYDVR0OBBYEFEosdONlsoPY5VyvU9YeRsjpsErQMB8GA1UdIwQY
+MBaAFEosdONlsoPY5VyvU9YeRsjpsErQMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggIBAFrDKCBfzU9nfNKLMhyK0Y9+/L+61yuCASxBiE+3rcdbm1pv
+eHQ3s+4Eize22ldaoLH9bjNErkxXwgQGMkMqTqDxHDEOprYufESIW5fmew5vpfRr
+7LWBsvm8zfByTm/sy2g9y9vgr6a/3WkuPGr/7+ZhWBwXmPn/cm/vtxCPhiJsKB8t
+m2jcoIgbdYZfOL4/0t1x6wJadJXNzrCOolUVfqxOPtjtuVRRNTUIhBaLlenfd+ot
+1Cgtq8GzS1EoR+0FnZzXOpgHlJ1LUGxHL86PhxLzV63tRGsBhC3Wt2FAmu8tem5X
+1AyXfUXV1Qn5BK6AE93m1s5UGMcVUDlbFMYK7quEuJHjcSxU07QpCKGrDQ276P2O
+CeuS72Yha4Rz4B5CaksIyB/CMHqz/VI2YK9cK10+tiz+KnwSaChsmUuNcjkL5f3l
+9rEW5ynZrO2duYNfDJRy7QgpSaj2Ac/6lcwL4BUCye/KFxdh2BNsldIIRJVfRurW
+LMPyM39s6tymNhthHHHhtq7YnFOjZGSK5jPNqWfgg0Hn9oxQ2UVvUnYdHPH6XVS3
+v+IToTme70l2ST3ARH0XMkrGf5L6pMquRXetxfpDjGR22CmVCYqX/XFPorv+2a5B
+tWwKUluRFrZGGPmKtURa04N9cFegXM2QV0sbWVr+Z9hb8rP3wlmrrB6iF+Z8
+-----END CERTIFICATE-----

echo-server.js

@@ -0,0 +1,31 @@
+// echo-server.js
+const http = require("http");
+const WebSocket = require("ws");
+
+// Use Railway’s assigned port or default to 80.
+const port = process.env.PORT || 80;
+
+// Create an HTTP server that can also handle upgrade requests.
+const server = http.createServer((req, res) => {
+	res.writeHead(200, { "Content-Type": "text/plain" });
+	res.end("Hello, this is an HTTP server that supports WebSocket upgrades.");
+});
+
+server.listen(port, "0.0.0.0", () => {
+	console.log(`HTTP/WebSocket server listening on port ${port}`);
+});
+
+// Bind the WebSocket server to the same HTTP server.
+const wss = new WebSocket.Server({ server });
+
+wss.on("connection", (ws) => {
+	console.log("Client connected");
+	ws.on("message", (message) => {
+		console.log(`Received: ${message}`);
+		ws.send(`Echo: ${message}`);
+	});
+});
+
+wss.on("error", (err) => {
+	console.error("Server error:", err);
+});

fly.toml

@@ -0,0 +1,29 @@
+# fly.toml app configuration file generated for muddy-river-1862 on 2025-03-16T02:26:49-04:00
+#
+# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
+#
+
+app = "muddy-river-1862"
+primary_region = "iad"
+
+[build]
+  dockerfile = "Dockerfile.fly"
+
+[http_service]
+  internal_port = 8888
+  force_https = true
+  auto_stop_machines = false
+  auto_start_machines = true
+  min_machines_running = 1
+  processes = ["app"]
+
+# Machine resources - increased memory to prevent OOM errors
+[[vm]]
+  cpu_kind = "shared"
+  cpus = 1
+  memory_mb = 4096
+
+# Volume mounts - using a single volume for all data
+[[mounts]]
+  source = "zarr_data"
+  destination = "/data"

key.pem

@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQC+Obd8Cwu7G1AP
+KWaAd6WqESqzQgF22PCIn8CQTfWSEvGixC7Ncz90NjVGa5mRLXSAtojQfbbv+Vqe
+WgL4Qwy5DU5wSIP4LTcJ0aWJ6EIP06rryIWPwOcwHwXJF9iyG6sKGqpOEO/qn5Lz
+fMsJhwWkybDiI5AydhQhwngh3tNLUCUszrHqiSmNergfypa/zOvD2tIs9qCkvKcW
+DgTwX9B637xVFU5DkXUViCaU7yl+z7WEWz070vssIoPffOwd42seO/nq/Y1AZWk2
+CRArILSr49Oi0P9BSPfsSG75R5Hxsj0SIpsUwcCJ0u2MKFeeohCRPBYwWC34B6rX
+/ke1V1Ln+Ri43KAJXmI1kPIf1IyPqKDGppSOtOt8L2/DWlBwodxMWaYEmUESdssK
+y+3nCXVgGtD3JUAoCJPBhbJeTPQwDJTDfKfr6sLEY1jiv7aEBfg4SQqh0J7qpjVj
+JB9r1jBEbZlSh3C+06wXfX4wr3LbHmmKOVT7XYSyG9FmzMBQo4HSXRR56QJz7mHU
+6UxFuGSvbu06A9CSZMft5hjq7s1J+kzEtBLY36+AHSHN26NN9d/jdc4OlKb65KXW
+amYX8ttdlPPoWNTnnK9qoA5fXlqEhvKJA32u3gUoCbVZV/zjinXKpC4DPQ8FV2my
+Tqjkc7wSdHJwJOattg+/Ffg3Je/onQIDAQABAoICAAEHozSxjPLa7ZuPtpsMC5j1
+s6pu8SSWX8IEd4s0JXVwBgKGEYqD2tv6yW6GEhPwCKGKmp7LhA25PeLK7S0Zh12i
+7RgMTmco7wtABQhPsfnc1/owxC0cjS7wTF1UPim+yXxcJxImQ+1ScjXNkep4DShN
+cMstcUCNz/saHaH7uE7NF0t1BGunbxLAehsHcU4diDhg6LwJPGqf5j9WzMvGoYe2
+0UFZDMBMNwj18QZZ47QZ+q575DL3827JrYkNU/LQUjpJJ/QHqNO0BL54lpcIKC/8
+ocLu9pVeoo/K4X5buW4NX8cF4EjqlEEEm7T9oG8kd49GwMATkqXupuY/2i/651wc
+ouVedFBWTWtT2L7bmvWMK0NZVUvlxLT7ftnvqBKFFGBE79zzgtec5FdT0NaQBtaK
+DDmi4b6u6RJK5IctZlrR4OHDT079gl6QqLLk1PG0kn9l2NjQN5VLRARcydEbL9QW
+VbCUKokJlzqEoGTfHHaKZdRyeGZtUkrnm6tknYKHln4hyyhM5iRLLK9ABGKVjyzK
+oqjw4bpLsgmCqughezKEk/hruZZCXPPA9K8pGfoO2dnhND79VN3RpcyQdP0w7Jlx
+J5WMJYZHn0l+5CgzTtaQupULincq2Ax9e1DbN8Lc5DnhoeI6fy2l9f4mdpVEXQTb
+H0Mpq0EgrLVjC6QasdUJAoIBAQDgntXn4DrZ9/+gdki0/LYxyEAM/XVnMeog7Ecj
+9AEFhtU4hWCliySeWDeDCKGt6/0+WK/BY81gK6Q8CKhT7vaws9wiDYh9Ye2e5MPI
+tnEK2H1fTVUEFnzOjORcw2TonXJuoZ7hIfPq3q2y7mnP0c7DrE7GzAEhrXA6xmpq
+FCyYWurCrtkywCiLR2rbMKtACTWfsIZ9+qYot36K4gVj4C0UQ/qOLpO0BPwmmse1
+6HO4BG2ud/mD2g9bGOmdEDe0zNQk5JCpBT1JXHSN5/9k7Q9vesHI9ghGbbXOSnfm
+NkcAwGR/Bo/+sTUunOXU1dhKH5uIWdpDDS0uE9nOw20YklqZAoIBAQDYzM2isPZY
+HWKK5ONj+vhBE2acrTr+NQVlqnlaWlRCNoeymJ6+ButZnNNiYZiY/UTl+TZTFnNW
+O2inSA81NBN43yjnFC8/a/zOwyx2Uu0co8AMTX2KEJwl3iqCY4ISPQPIuheAJp6H
+DkggkgkMDs4++G+ExqsczycWbJvt++ikGbBqtePwqoJYw1k1Pu+n0ID41rmww+i8
+526cpr7H9Z6Hd36cmwqoAC6tEF+G1E0Hdg6s84cSVpBRd9RGYUf87+reheqQhBxC
+KZ/AbskcnuBjuAJ0Dhks7mUNKYE2ILUlEcs+OJXNKtFBwL5wEpkbmT6SzMWW/Y3i
+axsOmfWKjiSlAoIBAQDJdUwOourFsgsgRNZjFUDnl13zCKRywK7nhmMr2N3Nsies
+gNu0vf5C0oY/TM+NqNnCQFKqFbQOrDWp7m3ikbHJvBcUp6SiJ+GyICWE8DSk8JRJ
+lJdKiky4CF+M32ayxvvqQ6S28PfO0JdjozddwCQ9FV62KCRfqym8XVb1+1Af+XMS
+1zVQsk3q1Uv+eUEuttPUfMOzW8oJPMIPGDaTZG2p/mpLeXTaAZqRmULDYU/gXIBu
++YXdfwdEwixMxU3hJ9Choioond1g3b5fxBMo1wGAD86cSiYT3LQx6FJvNIEsKGI6
+F42wdgln3Cq6YG2T1ZNukIAUoU3DROSV8Gr5ndIJAoIBAQC8H1sApVXduAHeTCcU
+hp4Muez8X0UleJSqV3bd9CvRGMg/LTfx5McsB21VnSJw/rReahW9m2mRlhOFtH2d
+yQZsiBhSC63D9sx0Dd/y/JpLk7v98O/Scf9MzK9OXVJsyQ0TjIb/6IL+5mGoAZBg
+KijFc4YxTMMGSKq+Rpg3tbN1UeVHc3XLS+m3ZCtwjKAgmkiPpKOUwyqdYKUWwYoi
+sPtSyVnQQqjIhv/5pI9/y5DbvJGXeenm+75XMwe72oZA8V54oPOdW0W6E6xPqtJQ
+R3ymivF2CxXFa0e8WBuLAG6vL/WQlEFIurLBv6KAVWMOE7e8YnvCPVp08kOUM/xg
+sl8RAoIBAQC5NKHyIOyvbMfBGuMhEl9FMyWLxhbt1UGoUHkBGT0BAawPsxYKTxTL
+189Waw8DRc1kyba7M2HA28+vj8VyKl+ceum+zHtOMtHfXQqnPj853OAE9VgF4wiU
+wadK8Ze8p4VhUDBAkUq4wEcjSSCVEExeX3UhFoY/aTIvmdNjyFqto+fstvE3dB8r
+P+uymSqWecEWqkAQ30tu6/aA3Hu6ja3k3sF/44Ly2kzQtzwY7jRXz87EKFjFf6L8
+C3cv83P8uLkMZYz+PTOXCZI9msVIIbwwyO8L1AhRmFimW1N5nvD8fL9Xq+flUYvD
+UKEKogWIwTzjESwJXkfIXPsBGqaQtNp6
+-----END PRIVATE KEY-----