diff --git a/llm/llama-3_1-finetuning/configs/70B-lora.yaml b/llm/llama-3_1-finetuning/configs/70B-lora.yaml deleted file mode 100644 index 612048536a3..00000000000 --- a/llm/llama-3_1-finetuning/configs/70B-lora.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# Config for multi-device LoRA in lora_finetune_distributed.py -# using a Llama3.1 70B model -# -# This config assumes that you've run the following command before launching -# this run: -# tune download meta-llama/Meta-Llama-3.1-70B-Instruct --output-dir /tmp/Meta-Llama-3.1-70B-Instruct --ignore-patterns "original/consolidated*" -# -# This config needs 8 GPUs to run -# tune run --nproc_per_node 8 lora_finetune_distributed --config llama3_1/70B_lora - -# Model Arguments -model: - _component_: torchtune.models.llama3_1.lora_llama3_1_70b - lora_attn_modules: ['q_proj', 'k_proj', 'v_proj'] - apply_lora_to_mlp: False - apply_lora_to_output: False - lora_rank: 16 - lora_alpha: 32 - -tokenizer: - _component_: torchtune.models.llama3.llama3_tokenizer - path: /tmp/Meta-Llama-3.1-70B-Instruct/original/tokenizer.model - -checkpointer: - _component_: torchtune.utils.FullModelHFCheckpointer - checkpoint_dir: /tmp/Meta-Llama-3.1-70B-Instruct/ - checkpoint_files: [ - model-00001-of-00030.safetensors, - model-00002-of-00030.safetensors, - model-00003-of-00030.safetensors, - model-00004-of-00030.safetensors, - model-00005-of-00030.safetensors, - model-00006-of-00030.safetensors, - model-00007-of-00030.safetensors, - model-00008-of-00030.safetensors, - model-00009-of-00030.safetensors, - model-00010-of-00030.safetensors, - model-00011-of-00030.safetensors, - model-00012-of-00030.safetensors, - model-00013-of-00030.safetensors, - model-00014-of-00030.safetensors, - model-00015-of-00030.safetensors, - model-00016-of-00030.safetensors, - model-00017-of-00030.safetensors, - model-00018-of-00030.safetensors, - model-00019-of-00030.safetensors, - model-00020-of-00030.safetensors, - model-00021-of-00030.safetensors, - model-00022-of-00030.safetensors, - model-00023-of-00030.safetensors, - model-00024-of-00030.safetensors, - model-00025-of-00030.safetensors, - model-00026-of-00030.safetensors, - model-00027-of-00030.safetensors, - model-00028-of-00030.safetensors, - model-00029-of-00030.safetensors, - model-00030-of-00030.safetensors, - ] - recipe_checkpoint: null - output_dir: /tmp/Meta-Llama-3.1-70B-Instruct/ - model_type: LLAMA3 -resume_from_checkpoint: False - -# Dataset and Sampler -dataset: - _component_: torchtune.datasets.alpaca_dataset -seed: null -shuffle: True -batch_size: 2 - -# Optimizer and Scheduler -optimizer: - _component_: torch.optim.AdamW - weight_decay: 0.01 - lr: 3e-4 -lr_scheduler: - _component_: torchtune.modules.get_cosine_schedule_with_warmup - num_warmup_steps: 100 - -loss: - _component_: torch.nn.CrossEntropyLoss - -# Training -epochs: 1 -max_steps_per_epoch: null -gradient_accumulation_steps: 1 - -# Logging -output_dir: /tmp/lora_finetune_output -metric_logger: - _component_: torchtune.utils.metric_logging.DiskLogger - log_dir: ${output_dir} -log_every_n_steps: 1 -log_peak_memory_stats: False - -# Environment -device: cuda -dtype: bf16 -enable_activation_checkpointing: True diff --git a/llm/llama-3_1-finetuning/configs/8B-lora.yaml b/llm/llama-3_1-finetuning/configs/8B-lora.yaml deleted file mode 100644 index d3e3be5af8e..00000000000 --- a/llm/llama-3_1-finetuning/configs/8B-lora.yaml +++ /dev/null @@ -1,83 +0,0 @@ -# Config for multi-device LoRA finetuning in lora_finetune_distributed.py -# using a Llama3.1 8B Instruct model -# -# This config assumes that you've run the following command before launching -# this run: -# tune download meta-llama/Meta-Llama-3.1-8B-Instruct --output-dir /tmp/Meta-Llama-3.1-8B-Instruct --ignore-patterns "original/consolidated.00.pth" -# -# To launch on 2 devices, run the following command from root: -# tune run --nproc_per_node 2 lora_finetune_distributed --config llama3_1/8B_lora -# -# You can add specific overrides through the command line. For example -# to override the checkpointer directory while launching training -# you can run: -# tune run --nproc_per_node 2 lora_finetune_distributed --config llama3_1/8B_lora checkpointer.checkpoint_dir= -# -# This config works best when the model is being fine-tuned on 2+ GPUs. -# For single device LoRA finetuning please use 8B_lora_single_device.yaml -# or 8B_qlora_single_device.yaml - -# Tokenizer -tokenizer: - _component_: torchtune.models.llama3.llama3_tokenizer - path: /tmp/Meta-Llama-3.1-8B-Instruct/original/tokenizer.model - -# Model Arguments -model: - _component_: torchtune.models.llama3_1.lora_llama3_1_8b - lora_attn_modules: ['q_proj', 'v_proj'] - apply_lora_to_mlp: False - apply_lora_to_output: False - lora_rank: 8 - lora_alpha: 16 - -checkpointer: - _component_: torchtune.utils.FullModelHFCheckpointer - checkpoint_dir: /tmp/Meta-Llama-3.1-8B-Instruct/ - checkpoint_files: [ - model-00001-of-00004.safetensors, - model-00002-of-00004.safetensors, - model-00003-of-00004.safetensors, - model-00004-of-00004.safetensors - ] - recipe_checkpoint: null - output_dir: /tmp/Meta-Llama-3.1-8B-Instruct/ - model_type: LLAMA3 -resume_from_checkpoint: False - -# Dataset and Sampler -dataset: - _component_: torchtune.datasets.alpaca_cleaned_dataset -seed: null -shuffle: True -batch_size: 2 - -# Optimizer and Scheduler -optimizer: - _component_: torch.optim.AdamW - weight_decay: 0.01 - lr: 3e-4 -lr_scheduler: - _component_: torchtune.modules.get_cosine_schedule_with_warmup - num_warmup_steps: 100 - -loss: - _component_: torch.nn.CrossEntropyLoss - -# Training -epochs: 1 -max_steps_per_epoch: null -gradient_accumulation_steps: 32 - -# Logging -output_dir: /tmp/lora_finetune_output -metric_logger: - _component_: torchtune.utils.metric_logging.DiskLogger - log_dir: ${output_dir} -log_every_n_steps: 1 -log_peak_memory_stats: False - -# Environment -device: cuda -dtype: bf16 -enable_activation_checkpointing: False diff --git a/llm/llama-3_1-finetuning/lora.yaml b/llm/llama-3_1-finetuning/lora.yaml index bd92229c10f..9fa1f310a1c 100644 --- a/llm/llama-3_1-finetuning/lora.yaml +++ b/llm/llama-3_1-finetuning/lora.yaml @@ -8,45 +8,40 @@ # # HF_TOKEN=xxx sky launch lora.yaml -c llama31-70 --secret HF_TOKEN --env MODEL_SIZE=70B +resources: + accelerators: A100:8 + envs: MODEL_SIZE: 8B - DATASET: "yahma/alpaca-cleaned" - # Change this to your own checkpoint bucket - CHECKPOINT_BUCKET_NAME: sky-llama-31-checkpoints -secrets: - HF_TOKEN: null # Pass with `--secret HF_TOKEN` in CLI + DATASET: yahma/alpaca-cleaned -resources: - accelerators: A100:8 - disk_tier: best - use_spot: true +secrets: + HF_TOKEN: # Passed via --secret HF_TOKEN file_mounts: - /configs: ./configs /output: - name: $CHECKPOINT_BUCKET_NAME + source: s3://my-skypilot-bucket # Change to your object storage bucket mode: MOUNT - # Optionally, specify the store to enforce to use one of the stores below: - # r2/azure/gcs/s3/cos - # store: r2 setup: | - pip install torch==2.4.0 torchvision - - # Install torch tune from source for the latest Llama-3.1 model - pip install git+https://github.com/pytorch/torchtune.git@58255001bd0b1e3a81a6302201024e472af05379 - # pip install torchtune - + uv venv + source .venv/bin/activate + uv pip install torch==2.9.1 torchvision torchao torchtune tune download meta-llama/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ - --hf-token $HF_TOKEN \ - --output-dir /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ - --ignore-patterns "original/consolidated*" + --hf-token $HF_TOKEN \ + --output-dir /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ + --ignore-patterns "original/consolidated*" run: | + source .venv/bin/activate + # Generated a platform-specific config + tune cp llama3_1/${MODEL_SIZE}_lora /tmp/${MODEL_SIZE}-lora.yaml + + # Run distributed LoRA fine-tuning across all GPUs tune run --nproc_per_node $SKYPILOT_NUM_GPUS_PER_NODE \ - lora_finetune_distributed \ - --config /configs/${MODEL_SIZE}-lora.yaml \ - dataset.source=$DATASET + lora_finetune_distributed \ + --config /tmp/${MODEL_SIZE}-lora.yaml \ + dataset.source=$DATASET # Remove the checkpoint files to save space, LoRA serving only needs the # adapter files. diff --git a/llm/llama-3_1-finetuning/readme.md b/llm/llama-3_1-finetuning/readme.md index d607ad3282b..a4e8015abb7 100644 --- a/llm/llama-3_1-finetuning/readme.md +++ b/llm/llama-3_1-finetuning/readme.md @@ -38,55 +38,50 @@ The finetuning job is packaged in a SkyPilot YAML. It can be launched on any of # # HF_TOKEN=xxx sky launch lora.yaml -c llama31 --secret HF_TOKEN # -# To finetune a 70B model: +# To finetune a 70B model: # # HF_TOKEN=xxx sky launch lora.yaml -c llama31-70 --secret HF_TOKEN --env MODEL_SIZE=70B +resources: + accelerators: A100:8 + envs: MODEL_SIZE: 8B - HF_TOKEN: - DATASET: "yahma/alpaca-cleaned" - # Change this to your own checkpoint bucket - CHECKPOINT_BUCKET_NAME: sky-llama-31-checkpoints + DATASET: yahma/alpaca-cleaned - -resources: - accelerators: A100:8 - disk_tier: best - use_spot: true +secrets: + HF_TOKEN: # Passed via --secret HF_TOKEN file_mounts: - /configs: ./configs /output: - name: $CHECKPOINT_BUCKET_NAME + source: s3://my-skypilot-bucket # Change to your object storage bucket mode: MOUNT - # Optionally, specify the store to enforce to use one of the stores below: - # r2/azure/gcs/s3/cos - # store: r2 setup: | - pip install torch torchvision - - # Install torch tune from source for the latest Llama 3.1 model - pip install git+https://github.com/pytorch/torchtune.git@58255001bd0b1e3a81a6302201024e472af05379 - # pip install torchtune - + uv venv + source .venv/bin/activate + uv pip install torch==2.9.1 torchvision torchao torchtune tune download meta-llama/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ - --hf-token $HF_TOKEN \ - --output-dir /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ - --ignore-patterns "original/consolidated*" + --hf-token $HF_TOKEN \ + --output-dir /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct \ + --ignore-patterns "original/consolidated*" run: | + source .venv/bin/activate + # Generated a platform-specific config + tune cp llama3_1/${MODEL_SIZE}_lora /tmp/${MODEL_SIZE}-lora.yaml + + # Run distributed LoRA fine-tuning across all GPUs tune run --nproc_per_node $SKYPILOT_NUM_GPUS_PER_NODE \ - lora_finetune_distributed \ - --config /configs/${MODEL_SIZE}-lora.yaml \ - dataset.source=$DATASET + lora_finetune_distributed \ + --config /tmp/${MODEL_SIZE}-lora.yaml \ + dataset.source=$DATASET # Remove the checkpoint files to save space, LoRA serving only needs the # adapter files. rm /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct/*.pt rm /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct/*.safetensors - + mkdir -p /output/$MODEL_SIZE-lora rsync -Pavz /tmp/Meta-Llama-3.1-${MODEL_SIZE}-Instruct /output/$MODEL_SIZE-lora cp -r /tmp/lora_finetune_output /output/$MODEL_SIZE-lora/ @@ -105,17 +100,13 @@ export HF_TOKEN=xxxx # It takes about 40 mins on 8 A100 GPUs to finetune a 8B # Llama3.1 model with LoRA on Alpaca dataset. -sky launch -c llama31 lora.yaml \ - --secret HF_TOKEN --env MODEL_SIZE=8B \ - --env CHECKPOINT_BUCKET_NAME="your-own-bucket-name" +sky launch -c llama31 lora.yaml --secret HF_TOKEN ``` To finetune a larger model with 70B parameters, you can simply change the parameters as below: ```bash -sky launch -c llama31-70 lora.yaml \ - --secret HF_TOKEN --env MODEL_SIZE=70B \ - --env CHECKPOINT_BUCKET_NAME="your-own-bucket-name" +sky launch -c llama31-70 lora.yaml --secret HF_TOKEN --env MODEL_SIZE=70B ``` **Finetuning Llama 3.1 405B**: Work in progress! If you want to follow the work, join the [SkyPilot community Slack](https://slack.skypilot.co/) for discussions. @@ -128,9 +119,7 @@ You can do so by specifying the huggingface path to your own dataset as followin # It takes about 1 hour on 8 A100 GPUs to finetune a 8B # Llama3.1 model with LoRA on finance dataset. sky launch -c llama31 lora.yaml \ - --secret HF_TOKEN --env MODEL_SIZE=8B \ - --env CHECKPOINT_BUCKET_NAME="your-own-bucket-name" \ - --env DATASET="gbharti/finance-alpaca" + --secret HF_TOKEN --env DATASET="gbharti/finance-alpaca" ```
@@ -150,7 +139,7 @@ With a finetuned Llama 3.1 trained on your dataset, you can now serve the finetu ```bash sky launch -c serve-llama31 serve.yaml \ --env LORA_NAME="my-finance-lora" \ - --env CHECKPOINT_BUCEKT_NAME="your-own-bucket-name" + --env CHECKPOINT_BUCKET_NAME="your-own-bucket-name" ``` You can interact with the model in a terminal: