RooseBERT/sh/run_mlm_scratch.sh at main · deborahdore/RooseBERT · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
#!/bin/bash
#SBATCH --job-name={job_name}
#SBATCH -C a100

#SBATCH --ntasks=8
#SBATCH --ntasks-per-node=8
#SBATCH --gres=gpu:8
#SBATCH --cpus-per-task=8

#SBATCH --time=20:00:00
#SBATCH --output=logs/{job_name}_%j.out
#SBATCH --error=logs/{job_name}_%j.out

#SBATCH --hint=nomultithread

conda activate roosebert

export TOKENIZERS_PARALLELISM=false
export WANDB_PROJECT="Masked_Language_Modelling"
export MASTER_PORT=6000
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
wandb offline

# ------------------ HYPERPARAMETERS ------------------
N_GPUS=8
TYPE="uncased"
MODEL_NAME="bert-base-$TYPE"
LOWERCASE=true

# FIRST TRAINING PHASE
MAX_STEPS_1=200000
MAX_SEQ_LEN_1=128
BATCH=64
GRAD_ACC=4
LR=3e-4

# SECOND TRAINING PHASE
MAX_STEPS_2=250000
MAX_SEQ_LEN_2=512

RUN_NAME="${MODEL_NAME}-scr-batch$((BATCH * N_GPUS * GRAD_ACC))-lr${LR}"

printf "Starting training run: %s\n" "$RUN_NAME"

mkdir -p "logs/${RUN_NAME}" "cache/${RUN_NAME}-max${MAX_STEPS_1}"

# ------------------ TRAINING PHASE 1 ------------------

python -m torch.distributed.launch --nproc_per_node=${N_GPUS} \
        --node_rank=${SLURM_PROCID} \
        --master_addr=${MASTER_ADDR} \
        --master_port=${MASTER_PORT} \
        --rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT} --rdzv_backend=c10d \
        src/run_mlm.py \
        --model_type "bert" \
        --tokenizer_name "./tokenizer_$TYPE" \
        --lowercase $LOWERCASE \
        --cache_dir "cache/${RUN_NAME}-max${MAX_STEPS_1}" \
        --train_file "data/training/max_128/train.csv" \
        --validation_file "data/training/max_128/dev.csv" \
        --max_seq_length "$MAX_SEQ_LEN_1" \
        --preprocessing_num_workers 8 \
        --output_dir "logs/$RUN_NAME/" \
        --overwrite_output_dir  \
        --do_train \
        --do_eval \
        --eval_strategy "steps" \
        --per_device_train_batch_size $BATCH \
        --per_device_eval_batch_size $BATCH \
        --gradient_accumulation_steps $GRAD_ACC \
        --learning_rate $LR \
        --weight_decay 0.01 \
        --adam_beta1 0.9 --adam_beta2 0.98 --adam_epsilon 1e-6 \
        --max_steps $MAX_STEPS_1 \
        --warmup_steps 10000 \
        --logging_dir "logs/$RUN_NAME/" \
        --logging_strategy "steps" \
        --logging_steps 1000 \
        --save_strategy "steps" \
        --save_steps 10000 \
        --save_total_limit 1 \
        --seed 42 \
        --data_seed 42 \
        --fp16 \
        --local_rank 0 \
        --eval_steps 2000 \
        --dataloader_num_workers 8 \
        --run_name "$RUN_NAME" \
        --deepspeed "configs/deepspeed_config.json" \
        --report_to "wandb" \
        --eval_on_start \
        --log_level "detail"

# ------------------ TRAINING PHASE 2 ------------------
CHECKPOINT_PATH="logs/${RUN_NAME}/checkpoint-$MAX_STEPS_1"

python -m torch.distributed.launch --nproc_per_node=${N_GPUS} \
        --node_rank=${SLURM_PROCID} \
        --master_addr=${MASTER_ADDR} \
        --master_port=${MASTER_PORT} \
        --rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT} --rdzv_backend=c10d \
        src/run_mlm.py \
        --model_name_or_path "$CHECKPOINT_PATH" \
        --tokenizer_name "./tokenizer_$TYPE" \
        --overwrite_output_dir  \
        --resume_from_checkpoint "$CHECKPOINT_PATH" \
        --cache_dir "cache/${RUN_NAME}-max${MAX_STEPS_2}" \
        --train_file "data/training/max_512/train.csv" \
        --validation_file "data/training/max_512/dev.csv" \
        --max_seq_length "$MAX_SEQ_LEN_2" \
        --preprocessing_num_workers 8 \
        --output_dir "logs/$RUN_NAME/" \
        --do_train \
        --do_eval \
        --eval_strategy "steps" \
        --per_device_train_batch_size $BATCH \
        --per_device_eval_batch_size $BATCH \
        --gradient_accumulation_steps $GRAD_ACC \
        --learning_rate $LR \
        --weight_decay 0.01 \
        --adam_beta1 0.9 --adam_beta2 0.98 --adam_epsilon 1e-6 \
        --max_steps $MAX_STEPS_2 \
        --logging_dir "logs/$RUN_NAME/" \
        --logging_strategy "steps" \
        --logging_steps 1000 \
        --save_strategy "steps" \
        --save_steps 10000 \
        --save_total_limit 1 \
        --seed 42 \
        --data_seed 42 \
        --fp16 \
        --local_rank 0 \
        --eval_steps 2000 \
        --dataloader_num_workers 8 \
        --run_name "$RUN_NAME" \
        --deepspeed "configs/deepspeed_config.json" \
        --report_to "wandb" \
        --eval_on_start \
        --log_level "detail"