|
| 1 | +# Copyright 2025 The HuggingFace Team. |
| 2 | +# |
| 3 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +# you may not use this file except in compliance with the License. |
| 5 | +# You may obtain a copy of the License at |
| 6 | +# |
| 7 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | +# |
| 9 | +# Unless required by applicable law or agreed to in writing, software |
| 10 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | +# See the License for the specific language governing permissions and |
| 13 | +# limitations under the License. |
| 14 | + |
| 15 | +import unittest |
| 16 | + |
| 17 | +import numpy as np |
| 18 | +import pytest |
| 19 | +import torch |
| 20 | +from PIL import Image |
| 21 | +from transformers import Qwen2_5_VLConfig, Qwen2_5_VLForConditionalGeneration, Qwen2Tokenizer, Qwen2VLProcessor |
| 22 | + |
| 23 | +from diffusers import ( |
| 24 | + AutoencoderKLQwenImage, |
| 25 | + FlowMatchEulerDiscreteScheduler, |
| 26 | + QwenImageEditPipeline, |
| 27 | + QwenImageTransformer2DModel, |
| 28 | +) |
| 29 | +from diffusers.utils.testing_utils import enable_full_determinism, torch_device |
| 30 | + |
| 31 | +from ..pipeline_params import TEXT_TO_IMAGE_PARAMS |
| 32 | +from ..test_pipelines_common import PipelineTesterMixin, to_np |
| 33 | + |
| 34 | + |
| 35 | +enable_full_determinism() |
| 36 | + |
| 37 | + |
| 38 | +class QwenImageEditPipelineFastTests(PipelineTesterMixin, unittest.TestCase): |
| 39 | + pipeline_class = QwenImageEditPipeline |
| 40 | + params = TEXT_TO_IMAGE_PARAMS - {"cross_attention_kwargs"} |
| 41 | + batch_params = frozenset(["prompt", "image"]) |
| 42 | + image_params = frozenset(["image"]) |
| 43 | + image_latents_params = frozenset(["latents"]) |
| 44 | + required_optional_params = frozenset( |
| 45 | + [ |
| 46 | + "num_inference_steps", |
| 47 | + "generator", |
| 48 | + "latents", |
| 49 | + "return_dict", |
| 50 | + "callback_on_step_end", |
| 51 | + "callback_on_step_end_tensor_inputs", |
| 52 | + ] |
| 53 | + ) |
| 54 | + supports_dduf = False |
| 55 | + test_xformers_attention = False |
| 56 | + test_layerwise_casting = True |
| 57 | + test_group_offloading = True |
| 58 | + |
| 59 | + def get_dummy_components(self): |
| 60 | + tiny_ckpt_id = "hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration" |
| 61 | + |
| 62 | + torch.manual_seed(0) |
| 63 | + transformer = QwenImageTransformer2DModel( |
| 64 | + patch_size=2, |
| 65 | + in_channels=16, |
| 66 | + out_channels=4, |
| 67 | + num_layers=2, |
| 68 | + attention_head_dim=16, |
| 69 | + num_attention_heads=3, |
| 70 | + joint_attention_dim=16, |
| 71 | + guidance_embeds=False, |
| 72 | + axes_dims_rope=(8, 4, 4), |
| 73 | + ) |
| 74 | + |
| 75 | + torch.manual_seed(0) |
| 76 | + z_dim = 4 |
| 77 | + vae = AutoencoderKLQwenImage( |
| 78 | + base_dim=z_dim * 6, |
| 79 | + z_dim=z_dim, |
| 80 | + dim_mult=[1, 2, 4], |
| 81 | + num_res_blocks=1, |
| 82 | + temperal_downsample=[False, True], |
| 83 | + latents_mean=[0.0] * z_dim, |
| 84 | + latents_std=[1.0] * z_dim, |
| 85 | + ) |
| 86 | + |
| 87 | + torch.manual_seed(0) |
| 88 | + scheduler = FlowMatchEulerDiscreteScheduler() |
| 89 | + |
| 90 | + torch.manual_seed(0) |
| 91 | + config = Qwen2_5_VLConfig( |
| 92 | + text_config={ |
| 93 | + "hidden_size": 16, |
| 94 | + "intermediate_size": 16, |
| 95 | + "num_hidden_layers": 2, |
| 96 | + "num_attention_heads": 2, |
| 97 | + "num_key_value_heads": 2, |
| 98 | + "rope_scaling": { |
| 99 | + "mrope_section": [1, 1, 2], |
| 100 | + "rope_type": "default", |
| 101 | + "type": "default", |
| 102 | + }, |
| 103 | + "rope_theta": 1000000.0, |
| 104 | + }, |
| 105 | + vision_config={ |
| 106 | + "depth": 2, |
| 107 | + "hidden_size": 16, |
| 108 | + "intermediate_size": 16, |
| 109 | + "num_heads": 2, |
| 110 | + "out_hidden_size": 16, |
| 111 | + }, |
| 112 | + hidden_size=16, |
| 113 | + vocab_size=152064, |
| 114 | + vision_end_token_id=151653, |
| 115 | + vision_start_token_id=151652, |
| 116 | + vision_token_id=151654, |
| 117 | + ) |
| 118 | + text_encoder = Qwen2_5_VLForConditionalGeneration(config) |
| 119 | + tokenizer = Qwen2Tokenizer.from_pretrained(tiny_ckpt_id) |
| 120 | + |
| 121 | + components = { |
| 122 | + "transformer": transformer, |
| 123 | + "vae": vae, |
| 124 | + "scheduler": scheduler, |
| 125 | + "text_encoder": text_encoder, |
| 126 | + "tokenizer": tokenizer, |
| 127 | + "processor": Qwen2VLProcessor.from_pretrained(tiny_ckpt_id), |
| 128 | + } |
| 129 | + return components |
| 130 | + |
| 131 | + def get_dummy_inputs(self, device, seed=0): |
| 132 | + if str(device).startswith("mps"): |
| 133 | + generator = torch.manual_seed(seed) |
| 134 | + else: |
| 135 | + generator = torch.Generator(device=device).manual_seed(seed) |
| 136 | + |
| 137 | + inputs = { |
| 138 | + "prompt": "dance monkey", |
| 139 | + "image": Image.new("RGB", (32, 32)), |
| 140 | + "negative_prompt": "bad quality", |
| 141 | + "generator": generator, |
| 142 | + "num_inference_steps": 2, |
| 143 | + "true_cfg_scale": 1.0, |
| 144 | + "height": 32, |
| 145 | + "width": 32, |
| 146 | + "max_sequence_length": 16, |
| 147 | + "output_type": "pt", |
| 148 | + } |
| 149 | + |
| 150 | + return inputs |
| 151 | + |
| 152 | + def test_inference(self): |
| 153 | + device = "cpu" |
| 154 | + |
| 155 | + components = self.get_dummy_components() |
| 156 | + pipe = self.pipeline_class(**components) |
| 157 | + pipe.to(device) |
| 158 | + pipe.set_progress_bar_config(disable=None) |
| 159 | + |
| 160 | + inputs = self.get_dummy_inputs(device) |
| 161 | + image = pipe(**inputs).images |
| 162 | + generated_image = image[0] |
| 163 | + self.assertEqual(generated_image.shape, (3, 32, 32)) |
| 164 | + |
| 165 | + # fmt: off |
| 166 | + expected_slice = torch.tensor([[0.5637, 0.6341, 0.6001, 0.5620, 0.5794, 0.5498, 0.5757, 0.6389, 0.4174, 0.3597, 0.5649, 0.4894, 0.4969, 0.5255, 0.4083, 0.4986]]) |
| 167 | + # fmt: on |
| 168 | + |
| 169 | + generated_slice = generated_image.flatten() |
| 170 | + generated_slice = torch.cat([generated_slice[:8], generated_slice[-8:]]) |
| 171 | + self.assertTrue(torch.allclose(generated_slice, expected_slice, atol=1e-3)) |
| 172 | + |
| 173 | + def test_inference_batch_single_identical(self): |
| 174 | + self._test_inference_batch_single_identical(batch_size=3, expected_max_diff=1e-1) |
| 175 | + |
| 176 | + def test_attention_slicing_forward_pass( |
| 177 | + self, test_max_difference=True, test_mean_pixel_difference=True, expected_max_diff=1e-3 |
| 178 | + ): |
| 179 | + if not self.test_attention_slicing: |
| 180 | + return |
| 181 | + |
| 182 | + components = self.get_dummy_components() |
| 183 | + pipe = self.pipeline_class(**components) |
| 184 | + for component in pipe.components.values(): |
| 185 | + if hasattr(component, "set_default_attn_processor"): |
| 186 | + component.set_default_attn_processor() |
| 187 | + pipe.to(torch_device) |
| 188 | + pipe.set_progress_bar_config(disable=None) |
| 189 | + |
| 190 | + generator_device = "cpu" |
| 191 | + inputs = self.get_dummy_inputs(generator_device) |
| 192 | + output_without_slicing = pipe(**inputs)[0] |
| 193 | + |
| 194 | + pipe.enable_attention_slicing(slice_size=1) |
| 195 | + inputs = self.get_dummy_inputs(generator_device) |
| 196 | + output_with_slicing1 = pipe(**inputs)[0] |
| 197 | + |
| 198 | + pipe.enable_attention_slicing(slice_size=2) |
| 199 | + inputs = self.get_dummy_inputs(generator_device) |
| 200 | + output_with_slicing2 = pipe(**inputs)[0] |
| 201 | + |
| 202 | + if test_max_difference: |
| 203 | + max_diff1 = np.abs(to_np(output_with_slicing1) - to_np(output_without_slicing)).max() |
| 204 | + max_diff2 = np.abs(to_np(output_with_slicing2) - to_np(output_without_slicing)).max() |
| 205 | + self.assertLess( |
| 206 | + max(max_diff1, max_diff2), |
| 207 | + expected_max_diff, |
| 208 | + "Attention slicing should not affect the inference results", |
| 209 | + ) |
| 210 | + |
| 211 | + def test_vae_tiling(self, expected_diff_max: float = 0.2): |
| 212 | + generator_device = "cpu" |
| 213 | + components = self.get_dummy_components() |
| 214 | + |
| 215 | + pipe = self.pipeline_class(**components) |
| 216 | + pipe.to("cpu") |
| 217 | + pipe.set_progress_bar_config(disable=None) |
| 218 | + |
| 219 | + # Without tiling |
| 220 | + inputs = self.get_dummy_inputs(generator_device) |
| 221 | + inputs["height"] = inputs["width"] = 128 |
| 222 | + output_without_tiling = pipe(**inputs)[0] |
| 223 | + |
| 224 | + # With tiling |
| 225 | + pipe.vae.enable_tiling( |
| 226 | + tile_sample_min_height=96, |
| 227 | + tile_sample_min_width=96, |
| 228 | + tile_sample_stride_height=64, |
| 229 | + tile_sample_stride_width=64, |
| 230 | + ) |
| 231 | + inputs = self.get_dummy_inputs(generator_device) |
| 232 | + inputs["height"] = inputs["width"] = 128 |
| 233 | + output_with_tiling = pipe(**inputs)[0] |
| 234 | + |
| 235 | + self.assertLess( |
| 236 | + (to_np(output_without_tiling) - to_np(output_with_tiling)).max(), |
| 237 | + expected_diff_max, |
| 238 | + "VAE tiling should not affect the inference results", |
| 239 | + ) |
| 240 | + |
| 241 | + @pytest.mark.xfail(condition=True, reason="Preconfigured embeddings need to be revisited.", strict=True) |
| 242 | + def test_encode_prompt_works_in_isolation(self, extra_required_param_value_dict=None, atol=1e-4, rtol=1e-4): |
| 243 | + super().test_encode_prompt_works_in_isolation(extra_required_param_value_dict, atol, rtol) |
0 commit comments