Skip to content

Commit 6b6fd54

Browse files
NathanBSCbuddh0
NathanBSC
authored and
buddh0
committed
support QA deploy
1 parent 636f92f commit 6b6fd54

File tree

9 files changed

+351
-17
lines changed

9 files changed

+351
-17
lines changed

‎.env‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ INIT_HOLDER="0x04d63aBCd2b9b1baa327f2Dda0f873F197ccd186"
55
INIT_HOLDER_PRV="59ba8068eb256d520179e903f43dacf6d8d57d72bd306e1bd603fdb8c8da10e8"
66
RPC_URL="http://127.0.0.1:8545"
77
GENESIS_COMMIT="020c0459e37d1f9d635c1cff86dd1099ab1383fa" # pascal commit
8-
PASSED_FORK_DELAY=40
8+
PASSED_FORK_DELAY=150
99
LAST_FORK_MORE_DELAY=10
1010
FullImmutabilityThreshold=512
1111
MinBlocksForBlobRequests=576
1212
DefaultExtraReserveForBlobRequests=32
1313
BreatheBlockInterval=600
1414
useLatestBscClient=false
15-
EnableSentryNode=false
16-
EnableFullNode=false
15+
EnableSentryNode=true
16+
EnableFullNode=true
1717
RegisterNodeID=false
1818
EnableEVNWhitelist=false

‎bsc_cluster.sh‎

Lines changed: 102 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,15 @@ basedir=$(
99
)
1010
workspace=${basedir}
1111
source ${workspace}/.env
12-
size=$((BSC_CLUSTER_SIZE))
12+
source ${workspace}/qa-env-resource/machines_meta.sh # including machine ips and ids, don't upload!!!
13+
validator_ips=(${validator_ips_comma//,/ })
14+
size=${#validator_ips[@]}
1315
stateScheme="hash"
1416
dbEngine="leveldb"
1517
gcmode="full"
1618
sleepBeforeStart=10
1719
sleepAfterStart=10
20+
copyDir="bsc-qa"
1821

1922
# stop geth client
2023
function exit_previous() {
@@ -55,7 +58,7 @@ function reset_genesis() {
5558
git reset --hard ${GENESIS_COMMIT}
5659
mv genesis-template.json.bk genesis-template.json
5760

58-
poetry install --no-root
61+
# poetry install --no-root
5962
npm install
6063
rm -rf lib/forge-std
6164
forge install --no-git --no-commit foundry-rs/forge-std@v1.7.3
@@ -81,28 +84,52 @@ function prepare_config() {
8184
mkdir -p ${targetDir} && cd ${targetDir}
8285
cp ${workspace}/keys/password.txt ./
8386
cp ${workspace}/.local/hardforkTime.txt ./
87+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
88+
sed -i -e "s/{{validatorAddr}}/${cons_addr}/g" chaind.sh && rm -f chaind.sh.bak
8489
bbcfee_addrs=${fee_addr}
8590
powers="0x000001d1a94a2000" #2000000000000
8691
mv ${workspace}/.local/bls${i}/bls ./ && rm -rf ${workspace}/.local/bls${i}
8792
vote_addr=0x$(cat ./bls/keystore/*json | jq .pubkey | sed 's/"//g')
8893
echo "${cons_addr},${bbcfee_addrs},${fee_addr},${powers},${vote_addr}" >> ${workspace}/genesis/validators.conf
8994
if [ ${EnableSentryNode} = true ]; then
90-
mkdir -p ${workspace}/.local/sentry${i}
95+
targetDir=${workspace}/.local/sentry${i}
96+
mkdir -p ${targetDir} && cd ${targetDir}
97+
cp ${workspace}/.local/hardforkTime.txt ./
98+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
99+
sed -i -e '/--mine/d' chaind.sh
100+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' chaind.sh
101+
sed -i -e 's/bin="bsc"/bin="sentry"/g' chaind.sh
102+
sed -i -e "s/portInc=0/portInc=2/g" chaind.sh
103+
rm -f chaind.sh.bak
104+
sed -i -e 's/workdir="validator"/workdir="sentry"/g' init.sh
105+
sed -i -e 's/bin="bsc"/bin="sentry"/g' init.sh
106+
rm -f init.sh.bak
107+
mv bsc.service sentry.service
108+
sed -i -e 's/validator/sentry/g' sentry.service
109+
sed -i -e 's/bsc/sentry/g' sentry.service
110+
rm -f sentry.service.bak
91111
fi
92112
done
93113
if [ ${EnableFullNode} = true ]; then
94-
mkdir -p ${workspace}/.local/fullnode0
114+
targetDir=${workspace}/.local/fullnode0
115+
mkdir -p ${targetDir} && cd ${targetDir}
116+
cp ${workspace}/.local/hardforkTime.txt ./
117+
cp ${workspace}/qa-env-resource/* ./ && rm -f upgrade-single*
118+
sed -i -e '/--mine/d' chaind.sh && rm -f chaind.sh.bak
95119
fi
96120
rm -f ${workspace}/.local/hardforkTime.txt
97121

98122
cd ${workspace}/genesis/
99123
git checkout HEAD contracts
100-
sed -i -e 's/alreadyInit = true;/turnLength = 4;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
124+
125+
# to test unregister relayer
126+
# sed -i -e 's/currentRelayers\[addr\] = true;/currentRelayers\[addr\] = true; relayersExistMap\[addr\] = true;/' ${workspace}/genesis/contracts/deprecated/RelayerHub.sol
127+
sed -i -e 's/alreadyInit = true;/turnLength = 8;alreadyInit = true;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
101128
sed -i -e 's/public onlyCoinbase onlyZeroGasPrice {/public onlyCoinbase onlyZeroGasPrice {if (block.number < 300) return;/' ${workspace}/genesis/contracts/BSCValidatorSet.sol
102-
103-
poetry run python -m scripts.generate generate-validators
104-
poetry run python -m scripts.generate generate-init-holders "${initHolders}"
105-
poetry run python -m scripts.generate dev \
129+
130+
python3 -m scripts.generate generate-validators
131+
python3 -m scripts.generate generate-init-holders "${initHolders}"
132+
python3 -m scripts.generate dev \
106133
--epoch "200" \
107134
--init-felony-slash-scope "60" \
108135
--breathe-block-interval "10 minutes" \
@@ -135,10 +162,10 @@ function initNetwork() {
135162

136163
init_extra_args=""
137164
if [ ${EnableSentryNode} = true ]; then
138-
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ports 30411"
165+
init_extra_args="--init.sentrynode-size ${size} --init.sentrynode-ips ${sentry_ips_comma}"
139166
fi
140167
if [ ${EnableFullNode} = true ]; then
141-
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ports 30511"
168+
init_extra_args="${init_extra_args} --init.fullnode-size 1 --init.fullnode-ips ${fullnode_ips_comma}"
142169
fi
143170
if [ "${RegisterNodeID}" = true ]; then
144171
if [ "${EnableSentryNode}" = true ]; then
@@ -154,10 +181,12 @@ function initNetwork() {
154181
init_extra_args="${init_extra_args} --init.evn-validator-whitelist"
155182
fi
156183
fi
157-
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --config ${workspace}/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
184+
${workspace}/bin/geth init-network --init.dir ${workspace}/.local --init.size=${size} --init.ips "${validator_ips_comma}" --config ${workspace}/qa-env-resource/config.toml ${init_extra_args} ${workspace}/genesis/genesis.json
158185
rm -f ${workspace}/*bsc.log*
159186
for ((i = 0; i < size; i++)); do
160187
sed -i -e '/"<nil>"/d' ${workspace}/.local/node${i}/config.toml
188+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/node${i}/config.toml
189+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/node${i}/config.toml
161190
mv ${workspace}/.local/validator${i}/keystore ${workspace}/.local/node${i}/ && rm -rf ${workspace}/.local/validator${i}
162191
# init genesis
163192
initLog=${workspace}/.local/node${i}/init.log
@@ -172,13 +201,17 @@ function initNetwork() {
172201

173202
if [ ${EnableSentryNode} = true ]; then
174203
sed -i -e '/"<nil>"/d' ${workspace}/.local/sentry${i}/config.toml
204+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/sentry${i}/config.toml
205+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/sentry${i}/config.toml
175206
initLog=${workspace}/.local/sentry${i}/init.log
176207
${workspace}/bin/geth --datadir ${workspace}/.local/sentry${i} init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
177208
rm -f ${workspace}/.local/sentry${i}/*bsc.log*
178209
fi
179210
done
180211
if [ ${EnableFullNode} = true ]; then
181212
sed -i -e '/"<nil>"/d' ${workspace}/.local/fullnode0/config.toml
213+
sed -i -e 's/:30311/:30611/g' ${workspace}/.local/fullnode0/config.toml
214+
sed -i -e 's/:30411/:30311/g' ${workspace}/.local/fullnode0/config.toml
182215
sed -i -e 's/EnableEVNFeatures = true/EnableEVNFeatures = false/g' ${workspace}/.local/fullnode0/config.toml
183216
initLog=${workspace}/.local/fullnode0/init.log
184217
${workspace}/bin/geth --datadir ${workspace}/.local/fullnode0 init --state.scheme path --db.engine pebble ${workspace}/genesis/genesis.json > "${initLog}" 2>&1
@@ -262,13 +295,55 @@ function native_start() {
262295

263296
function register_stakehub(){
264297
# wait feynman enable
265-
sleep 45
298+
sleep 100
266299
for ((i = 0; i < size; i++));do
267300
${workspace}/create-validator/create-validator --consensus-key-dir ${workspace}/keys/validator${i} --vote-key-dir ${workspace}/keys/bls${i} \
268301
--password-path ${workspace}/keys/password.txt --amount 20001 --validator-desc Val${i} --rpc-url ${RPC_URL}
269302
done
270303
}
271304

305+
function remote_start() {
306+
rm -rf /mnt/efs/${copyDir}/clusterNetwork
307+
cp -r ${workspace}/.local /mnt/efs/${copyDir}/clusterNetwork
308+
for dst_id in ${ips2ids[@]}; do
309+
if [ ${EnableSentryNode} = true ]; then
310+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service sentry stop"
311+
fi
312+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo service bsc stop"
313+
done
314+
sleep 100
315+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
316+
ips=(${validator_ips_comma//,/ })
317+
for ((i=0;i<${#ips[@]};i++));do
318+
dst_id=${ips2ids[${ips[i]}]}
319+
if [ ${EnableSentryNode} = true ]; then
320+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/sentry${i}/init.sh"
321+
fi
322+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/node${i}/init.sh"
323+
done
324+
if [ ${EnableFullNode} = true ]; then
325+
fullnode_ips=(${fullnode_ips_comma//,/ })
326+
dst_id=${ips2ids[${fullnode_ips[0]}]}
327+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" --parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo bash -x /mnt/efs/${copyDir}/clusterNetwork/fullnode0/init.sh"
328+
fi
329+
}
330+
331+
function remote_upgrade() {
332+
cp ${workspace}/bin/geth /mnt/efs/${copyDir}/clusterNetwork/
333+
if [ ${EnableSentryNode} = true ]; then
334+
cp ${workspace}/qa-env-resource/upgrade-single-sentry.sh /mnt/efs/${copyDir}/clusterNetwork/
335+
fi
336+
cp ${workspace}/qa-env-resource/upgrade-single-validator.sh /mnt/efs/${copyDir}/clusterNetwork/
337+
for dst_id in ${ips2ids[@]}; do
338+
if [ ${EnableSentryNode} = true ]; then
339+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
340+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-sentry.sh /tmp/ && sudo bash -x /tmp/upgrade-single-sentry.sh"
341+
fi
342+
aws ssm send-command --instance-ids "${dst_id}" --document-name "AWS-RunShellScript" \
343+
--parameters commands="sudo cp /mnt/efs/${copyDir}/clusterNetwork/geth /tmp/geth && sudo cp /mnt/efs/${copyDir}/clusterNetwork/upgrade-single-validator.sh /tmp/ && sudo bash -x /tmp/upgrade-single-validator.sh"
344+
done
345+
}
346+
272347
CMD=$1
273348
ValidatorIdx=$2
274349
case ${CMD} in
@@ -295,7 +370,20 @@ restart)
295370
exit_previous $ValidatorIdx
296371
native_start $ValidatorIdx
297372
;;
373+
remote_reset)
374+
create_validator
375+
reset_genesis
376+
prepare_config
377+
initNetwork
378+
remote_start
379+
# to prevent stuck
380+
remote_upgrade
381+
register_stakehub
382+
;;
383+
remote_upgrade)
384+
remote_upgrade
385+
;;
298386
*)
299-
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]"
387+
echo "Usage: bsc_cluster.sh | reset | stop [vidx]| start [vidx]| restart [vidx]| remote_reset | remote_upgrade"
300388
;;
301389
esac

‎qa-env-resource/bsc.service‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[Unit]
2+
Description=bsc
3+
After=network.target
4+
5+
[Service]
6+
Type=simple
7+
User=root
8+
Group=root
9+
ExecStart=/server/validator/chaind.sh -start
10+
ExecReload=/server/validator/chaind.sh -restart
11+
ExecStop=/server/validator/chaind.sh -stop
12+
PrivateTmp=true
13+
LimitNOFILE=10000
14+
StartLimitInterval=0
15+
16+
[Install]
17+
WantedBy=multi-user.target

‎qa-env-resource/chaind.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
export GOGC=200
6+
# default values
7+
workdir="validator"
8+
bin="bsc"
9+
portInc=0
10+
FullImmutabilityThreshold=90000
11+
MinBlocksForBlobRequests=524288
12+
DefaultExtraReserveForBlobRequests=28800
13+
BreatheBlockInterval=600
14+
LAST_FORK_MORE_DELAY=200
15+
16+
function startChaind() {
17+
workspace=/server/${workdir}
18+
19+
PassedForkTime=`cat ${workspace}/hardforkTime.txt|grep passedHardforkTime|awk -F" " '{print $NF}'`
20+
LastHardforkTime=$(expr ${PassedForkTime} + ${LAST_FORK_MORE_DELAY})
21+
initLog=${workspace}/init.log
22+
rialtoHash=`cat ${initLog}|grep "database=chaindata"|awk -F"=" '{print $NF}'|awk -F'"' '{print $1}'`
23+
24+
ip=`ifconfig eth0|grep inet|grep -v inet6 |awk '{ print $2 }'`
25+
sed -i -e "s?FileRoot = \"\"?FileRoot = \"/mnt/efs/${workdir}/${ip}/\"?g" /server/${workdir}/config.toml
26+
mkdir -p /mnt/efs/${workdir}/${ip}
27+
HTTPPort=$((8545 + ${portInc}))
28+
WSPort=${HTTPPort}
29+
MetricsPort=$((6060 + ${portInc}))
30+
PProfPort=$((${MetricsPort} + 1))
31+
${workspace}/${bin} --config ${workspace}/config.toml \
32+
--mine --vote --unlock {{validatorAddr}} --miner.etherbase {{validatorAddr}} --password ${workspace}/password.txt --blspassword ${workspace}/password.txt \
33+
--datadir ${workspace} \
34+
--rpc.allow-unprotected-txs --allow-insecure-unlock \
35+
--ws --ws.addr ${ip} --ws.port ${WSPort} --http.addr 0.0.0.0 --http.port ${HTTPPort} --http.corsdomain "*" \
36+
--metrics --metrics.addr 0.0.0.0 --metrics.port ${MetricsPort} \
37+
--pprof --pprof.port ${PProfPort} \
38+
--syncmode full --monitor.maliciousvote \
39+
--cache 10480 \
40+
--rialtohash ${rialtoHash} --override.passedforktime ${PassedForkTime} --override.lorentz ${PassedForkTime} --override.maxwell ${LastHardforkTime} \
41+
--override.immutabilitythreshold ${FullImmutabilityThreshold} --override.breatheblockinterval ${BreatheBlockInterval} \
42+
--override.minforblobrequest ${MinBlocksForBlobRequests} --override.defaultextrareserve ${DefaultExtraReserveForBlobRequests} \
43+
>> /mnt/efs/${workdir}/${ip}/bscnode.log 2>&1
44+
}
45+
46+
function stopChaind() {
47+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
48+
if [ -n "$pid" ]; then
49+
for((i=1;i<=4;i++));
50+
do
51+
kill $pid
52+
sleep 5
53+
pid=`ps -ef | grep /server/${workdir}/${bin} | grep -v grep | awk '{print $2}'`
54+
if [ -z "$pid" ]; then
55+
break
56+
elif [ $i -eq 4 ]; then
57+
kill -9 $pid
58+
fi
59+
done
60+
fi
61+
}
62+
63+
CMD=$1
64+
65+
case $CMD in
66+
-start)
67+
echo "start"
68+
startChaind
69+
;;
70+
-stop)
71+
echo "stop"
72+
stopChaind
73+
;;
74+
-restart)
75+
stopChaind
76+
sleep 3
77+
startChaind
78+
;;
79+
*)
80+
echo "Usage: chaind.sh -start | -stop | -restart .Or use systemctl start | stop | restart ${bin}.service "
81+
;;
82+
esac

0 commit comments

Comments
 (0)