Skip to content

Commit a88a109

Browse files
jyeshejyeshe
committed
Adds a script for local dev clusters
1 parent 25c4ace commit a88a109

File tree

4 files changed

+254
-49
lines changed

4 files changed

+254
-49
lines changed

‎bin/local_cluster‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
#!/usr/bin/env bash
2+
3+
COOKIE="rate-limiter-secret"
4+
5+
# Function to show usage
6+
show_usage() {
7+
echo "Usage:"
8+
echo " $0 [--proxy] --count <number> Start local cluster (default: 2 instances)"
9+
echo " $0 connect <node_number> Connect to a specific node (1-4)"
10+
echo ""
11+
echo "Options:"
12+
echo " --proxy Start a Caddy reverse proxy on port 4000 (nodes will start from 4001)"
13+
echo " --count <num> Number of nodes to start (1-4, default: 2)"
14+
exit 1
15+
}
16+
17+
# Handle connect subcommand
18+
if [ "$1" = "connect" ]; then
19+
if [ -z "$2" ] || ! [[ "$2" =~ ^[1-4]$ ]]; then
20+
echo "Error: Please specify a valid node number (1-4)"
21+
show_usage
22+
fi
23+
24+
NODE_NUM=$2
25+
echo "Connecting to node${NODE_NUM}@127.0.0.1..."
26+
exec iex --name "remote_shell${NODE_NUM}@127.0.0.1" --cookie "${COOKIE}" --remsh "node${NODE_NUM}@127.0.0.1"
27+
# The exec command replaces the current process, so we don't need an explicit exit
28+
# If we reach this point, it means the exec failed, so we'll exit with its status code
29+
exit $?
30+
fi
31+
32+
# Parse arguments
33+
USE_PROXY=false
34+
INSTANCES=2
35+
36+
while [[ $# -gt 0 ]]; do
37+
case $1 in
38+
--proxy)
39+
USE_PROXY=true
40+
shift
41+
;;
42+
--count)
43+
if [ -z "$2" ] || ! [[ "$2" =~ ^[0-9]+$ ]]; then
44+
echo "Error: --count requires a numeric argument"
45+
show_usage
46+
fi
47+
INSTANCES=$2
48+
shift 2
49+
;;
50+
*)
51+
echo "Unknown argument: $1"
52+
show_usage
53+
;;
54+
esac
55+
done
56+
57+
# Validate number of instances
58+
if ! [[ "$INSTANCES" =~ ^[0-9]+$ ]]; then
59+
echo "Error: Number of instances must be a positive integer"
60+
show_usage
61+
fi
62+
63+
if [ "$INSTANCES" -lt 1 ] || [ "$INSTANCES" -gt 4 ]; then
64+
echo "Error: Number of instances must be between 1 and 4"
65+
show_usage
66+
fi
67+
68+
# Check for Caddy if proxy is requested
69+
if [ "$USE_PROXY" = true ]; then
70+
if ! command -v caddy &>/dev/null; then
71+
echo "Error: Caddy is required for proxy mode but it's not installed"
72+
echo "Please install Caddy first:"
73+
echo " Mac: brew install caddy"
74+
echo " Linux: sudo apt install caddy"
75+
echo " Or visit: https://caddyserver.com/docs/install"
76+
exit 1
77+
fi
78+
fi
79+
80+
# Array to store background PIDs
81+
declare -a PIDS
82+
83+
# Colors for different processes
84+
declare -a COLORS=(
85+
"\033[0;36m" # Cyan
86+
"\033[0;32m" # Green
87+
"\033[0;35m" # Purple
88+
"\033[0;33m" # Yellow
89+
"\033[0;37m" # Gray (for proxy)
90+
)
91+
RESET="\033[0m"
92+
93+
# Cleanup function to kill all child processes
94+
cleanup() {
95+
echo "Shutting down all processes..."
96+
for pid in "${PIDS[@]}"; do
97+
kill "$pid" 2>/dev/null
98+
done
99+
exit 0
100+
}
101+
102+
# Set up trap for cleanup
103+
trap cleanup INT TERM
104+
105+
# Function to run a command with colored output
106+
run_with_color() {
107+
local color=$1
108+
local prefix=$2
109+
shift 2
110+
# Run the command and color its output
111+
"$@" 2>&1 | while read -r line; do
112+
echo -e "${color}${prefix} | ${line}${RESET}"
113+
done
114+
}
115+
116+
# Create Caddy configuration if proxy is enabled
117+
if [ "$USE_PROXY" = true ]; then
118+
BASE_PORT=4001
119+
CADDY_CONFIG=$(mktemp)
120+
echo "Creating Caddy configuration..."
121+
cat >"$CADDY_CONFIG" <<EOF
122+
# Global options
123+
{
124+
admin off
125+
auto_https off
126+
http_port 4000
127+
}
128+
129+
# Reverse proxy configuration
130+
localhost:4000 {
131+
reverse_proxy {
132+
to $(for i in $(seq 1 "$INSTANCES"); do echo "localhost:$((BASE_PORT + i - 1))"; done | paste -sd " " -)
133+
lb_policy round_robin
134+
}
135+
}
136+
EOF
137+
138+
# Only log Caddy config if LOG_LEVEL is debug
139+
if [ "${LOG_LEVEL:-}" = "debug" ]; then
140+
echo "Caddy config:"
141+
cat "$CADDY_CONFIG"
142+
fi
143+
144+
# Start Caddy
145+
run_with_color "${COLORS[4]}" "proxy" caddy run --adapter caddyfile --config "$CADDY_CONFIG" &
146+
PIDS+=($!)
147+
148+
# Cleanup Caddy config on exit
149+
trap 'rm -f "$CADDY_CONFIG"' EXIT
150+
151+
echo "Started reverse proxy on port 4000"
152+
else
153+
BASE_PORT=4000
154+
fi
155+
156+
# Start the requested number of instances
157+
for i in $(seq 1 "$INSTANCES"); do
158+
export RTM_PORT=$((2222 + i - 1)) PORT=$((BASE_PORT + i - 1))
159+
run_with_color "${COLORS[$i - 1]}" "node$i" elixir --cookie "${COOKIE}" --name "node$i@127.0.0.1" -S mix phx.server &
160+
PIDS+=($!)
161+
done
162+
163+
if [ "$USE_PROXY" = true ]; then
164+
echo "Started $INSTANCES node(s) on ports $((BASE_PORT))-$((BASE_PORT + INSTANCES - 1)) with load balancer on port 4000"
165+
echo "RTM ports: 2222-$((2222 + INSTANCES - 1))"
166+
else
167+
echo "Started $INSTANCES node(s) on ports $((BASE_PORT))-$((BASE_PORT + INSTANCES - 1))"
168+
echo "RTM ports: 2222-$((2222 + INSTANCES - 1))"
169+
fi
170+
echo "To connect to a specific node, use: $0 connect <node_number>"
171+
172+
# Wait for all background processes
173+
wait

‎config/dev.exs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ config :lightning, Lightning.Vault,
4848

4949
config :lightning, Lightning.Runtime.RuntimeManager, start: true
5050

51+
config :lightning, Lightning.DistributedRateLimiter, start: true
52+
5153
config :lightning, :workers,
5254
private_key: """
5355
-----BEGIN PRIVATE KEY-----

‎lib/lightning/config/bootstrap.ex‎

Lines changed: 73 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,8 @@ defmodule Lightning.Config.Bootstrap do
114114
"RTM",
115115
&Utils.ensure_boolean/1,
116116
Utils.get_env([:lightning, Lightning.Runtime.RuntimeManager, :start])
117-
)
117+
),
118+
port: env!("RTM_PORT", :integer, 2222)
118119

119120
config :lightning, :workers,
120121
private_key:
@@ -407,6 +408,10 @@ defmodule Lightning.Config.Bootstrap do
407408
config :logger, :level, log_level
408409
end
409410

411+
if log_level == :debug do
412+
config :libcluster, debug: true
413+
end
414+
410415
database_url = env!("DATABASE_URL", :string, nil)
411416

412417
config :lightning, Lightning.Repo,
@@ -417,7 +422,6 @@ defmodule Lightning.Config.Bootstrap do
417422
queue_interval: env!("DATABASE_QUEUE_INTERVAL", :integer, 1000)
418423

419424
host = env!("URL_HOST", :string, "example.com")
420-
port = env!("PORT", :integer, 4000)
421425
url_port = env!("URL_PORT", :integer, 443)
422426

423427
config :lightning,
@@ -463,18 +467,6 @@ defmodule Lightning.Config.Bootstrap do
463467
You can generate one by calling: mix phx.gen.secret
464468
"""
465469

466-
listen_address =
467-
env!(
468-
"LISTEN_ADDRESS",
469-
fn address ->
470-
address
471-
|> String.split(".")
472-
|> Enum.map(&String.to_integer/1)
473-
|> List.to_tuple()
474-
end,
475-
{127, 0, 0, 1}
476-
)
477-
478470
origins =
479471
env!(
480472
"ORIGINS",
@@ -489,40 +481,10 @@ defmodule Lightning.Config.Bootstrap do
489481

490482
url_scheme = env!("URL_SCHEME", :string, "https")
491483

492-
idle_timeout =
493-
env!(
494-
"IDLE_TIMEOUT",
495-
fn str ->
496-
case Integer.parse(str) do
497-
:error -> 60_000
498-
{val, _} -> val * 1_000
499-
end
500-
end,
501-
60_000
502-
)
503-
504484
config :lightning, LightningWeb.Endpoint,
505485
url: [host: host, port: url_port, scheme: url_scheme],
506486
secret_key_base: secret_key_base,
507487
check_origin: origins,
508-
http: [
509-
ip: listen_address,
510-
port: port,
511-
compress: true,
512-
protocol_options: [
513-
# Note that if a request is more than 10x the max dataclip size, we cut
514-
# the connection immediately to prevent memory issues via the
515-
# :max_skip_body_length setting.
516-
max_skip_body_length:
517-
Application.get_env(
518-
:lightning,
519-
:max_dataclip_size_bytes,
520-
10_000_000
521-
) *
522-
10,
523-
idle_timeout: idle_timeout
524-
]
525-
],
526488
server: true
527489
end
528490

@@ -538,6 +500,8 @@ defmodule Lightning.Config.Bootstrap do
538500
assert_receive_timeout: env!("ASSERT_RECEIVE_TIMEOUT", :integer, 1000)
539501
end
540502

503+
config :lightning, LightningWeb.Endpoint, http: http_config(config_env())
504+
541505
config :sentry,
542506
dsn: env!("SENTRY_DSN", :string, nil),
543507
filter: Lightning.SentryEventFilter,
@@ -811,4 +775,69 @@ defmodule Lightning.Config.Bootstrap do
811775
value -> value
812776
end
813777
end
778+
779+
defp http_config(env, opts \\ [])
780+
# Production environment configuration
781+
defp http_config(:prod, opts) do
782+
port = Keyword.get(opts, :port) || env!("PORT", :integer, 4000)
783+
784+
listen_address =
785+
env!(
786+
"LISTEN_ADDRESS",
787+
fn address ->
788+
address
789+
|> String.split(".")
790+
|> Enum.map(&String.to_integer/1)
791+
|> List.to_tuple()
792+
end,
793+
{127, 0, 0, 1}
794+
)
795+
796+
idle_timeout =
797+
env!(
798+
"IDLE_TIMEOUT",
799+
fn str ->
800+
case Integer.parse(str) do
801+
:error -> 60_000
802+
{val, _} -> val * 1_000
803+
end
804+
end,
805+
60_000
806+
)
807+
808+
[
809+
ip: listen_address,
810+
port: port,
811+
compress: true,
812+
protocol_options: [
813+
# Note that if a request is more than 10x the max dataclip size, we cut
814+
# the connection immediately to prevent memory issues via the
815+
# :max_skip_body_length setting.
816+
max_skip_body_length:
817+
Application.get_env(
818+
:lightning,
819+
:max_dataclip_size_bytes,
820+
10_000_000
821+
) * 10,
822+
idle_timeout: idle_timeout
823+
]
824+
]
825+
end
826+
827+
# Default configuration for non-production environments
828+
defp http_config(_env, opts) do
829+
port =
830+
Keyword.get(opts, :port) ||
831+
env!(
832+
"PORT",
833+
:integer,
834+
get_env(:lightning, [LightningWeb.Endpoint, :http, :port])
835+
)
836+
837+
[
838+
ip: {0, 0, 0, 0},
839+
port: port,
840+
compress: true
841+
]
842+
end
814843
end

‎test/lightning/distributed_rate_limiter_test.exs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,12 @@ defmodule Lightning.DistributedRateLimiterTest do
5858
assert 500 < wait_ms and wait_ms <= 1_000
5959
end
6060

61-
# For testing the replication use manual procedure or use this test case isolated to avoid interfering the tests above:
62-
# 0. Disable Endpoint server
63-
# 1. Run node1 on one terminal: iex --sname node1@localhost --cookie hordecookie -S mix phx.server
64-
# 2. Run node2 on another terminal: iex --sname node2@localhost --cookie hordecookie -S mix phx.server
65-
# 3. Call Lightning.DistributedRateLimiter.inspect_table() on both iex and they show the same ets table process and node.
61+
# Run this distributed integration test case separately to avoid interfering with the tests above.
62+
# For testing the replication use `bin/local_cluster` on the shell:
63+
# 1. In one shell run `./bin/local_cluster`
64+
# 2. On another shell run `./bin/local_cluster connect 2`
65+
# 3. Type `Lightning.DistributedRateLimiter.inspect_table()` to see that the ETS table is distributed
66+
# (on node1 or vice-versa if it was spawned on node2 when you connect to node 1).
6667
@tag :dist_integration
6768
test "works on top of a single worker of a distributed dynamic supervisor" do
6869
{:ok, peer, _node1, node2} = start_nodes(:node1, :node2, ~c"localhost")

0 commit comments

Comments
 (0)