ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)

=========================== short test summary info ============================
FAILED tests/test_grpo_trainer.py::GRPOTrainerTester::test_training_peft_with_gradient_checkpointing - ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)
FAILED tests/test_grpo_trainer.py::GRPOTrainerTester::test_training_scale_rewards_0 - ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)
FAILED tests/test_grpo_trainer.py::GRPOTrainerTester::test_training_scale_rewards_1_group - ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)

...

FAILED tests/test_rloo_trainer.py::RLOOTrainerTester::test_training_peft - ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)
FAILED tests/test_rloo_trainer.py::RLOOTrainerTester::test_training_peft_with_gradient_checkpointing - ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)
= 71 failed, 840 passed, 46 skipped, 145 warnings, 356 rerun in 683.28s (0:11:23) =

 _______ RLOOTrainerTester.test_training_peft_with_gradient_checkpointing _______
[gw1] linux -- Python 3.12.11 /__w/trl/trl/.venv/bin/python3

self = <tests.test_rloo_trainer.RLOOTrainerTester testMethod=test_training_peft_with_gradient_checkpointing>

    @require_peft
    def test_training_peft_with_gradient_checkpointing(self):
        """Test that training works with PEFT and gradient checkpointing enabled."""
        dataset = load_dataset("trl-internal-testing/zen", "standard_prompt_only", split="train")
    
        model = AutoModelForCausalLM.from_pretrained(
            "trl-internal-testing/tiny-Qwen2ForCausalLM-2.5",
            dtype=torch.float32,  # Use float32 for testing to avoid precision issues
        )
    
        lora_config = LoraConfig(
            r=8, lora_alpha=32, target_modules=["q_proj", "v_proj"], lora_dropout=0.05, bias="none"
        )
    
        training_args = RLOOConfig(
            output_dir=self.tmp_dir,
            learning_rate=0.1,
            per_device_train_batch_size=3,
            num_generations=3,
            max_completion_length=8,
            gradient_checkpointing=True,  # Enable gradient checkpointing
            report_to="none",
        )
>       trainer = RLOOTrainer(
            model=model,
            reward_funcs="trl-internal-testing/tiny-Qwen2ForSequenceClassification-2.5",
            args=training_args,
            train_dataset=dataset,
            peft_config=lora_config,
        )

tests/test_rloo_trainer.py:186: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
trl/trainer/rloo_trainer.py:359: in __init__
    reward_funcs[i] = AutoModelForSequenceClassification.from_pretrained(
.venv/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py:549: in from_pretrained
    config, kwargs = AutoConfig.from_pretrained(
.venv/lib/python3.12/site-packages/transformers/models/auto/configuration_auto.py:1346: in from_pretrained
    return config_class.from_dict(config_dict, **unused_kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.venv/lib/python3.12/site-packages/transformers/configuration_utils.py:808: in from_dict
    config = cls(**config_dict)
             ^^^^^^^^^^^^^^^^^^
.venv/lib/python3.12/site-packages/transformers/models/qwen2/configuration_qwen2.py:210: in __init__
    layer_type_validation(self.layer_types, self.num_hidden_layers)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

layer_types = ['full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', 'full_attention', ...]
num_hidden_layers = 2

    def layer_type_validation(layer_types: list[str], num_hidden_layers: Optional[int] = None):
        """Check that `layer_types` is correctly defined."""
        if not all(layer_type in ALLOWED_LAYER_TYPES for layer_type in layer_types):
            raise ValueError(f"The `layer_types` entries must be in {ALLOWED_LAYER_TYPES}")
        if num_hidden_layers is not None and num_hidden_layers != len(layer_types):
>           raise ValueError(
                f"`num_hidden_layers` ({num_hidden_layers}) must be equal to the number of layer types "
                f"({len(layer_types)})"
            )
E           ValueError: `num_hidden_layers` (2) must be equal to the number of layer types (64)

.venv/lib/python3.12/site-packages/transformers/configuration_utils.py:1377: ValueError