From ae47c45eb244208b9221ddf1ee74c26ef2691d7b Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Fri, 31 Oct 2025 20:01:38 +0000 Subject: [PATCH 1/2] update --- .../quantization_w8a8_fp8/kimi_linear_fp8.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 examples/quantization_w8a8_fp8/kimi_linear_fp8.py diff --git a/examples/quantization_w8a8_fp8/kimi_linear_fp8.py b/examples/quantization_w8a8_fp8/kimi_linear_fp8.py new file mode 100644 index 000000000..15b73d34a --- /dev/null +++ b/examples/quantization_w8a8_fp8/kimi_linear_fp8.py @@ -0,0 +1,43 @@ +from transformers import AutoProcessor, AutoModelForCausalLM + +from llmcompressor import oneshot +from llmcompressor.modifiers.quantization import QuantizationModifier +from llmcompressor.utils import dispatch_for_generation + +MODEL_ID = "//proving-grounds/engine/hub_cache/models--moonshotai--Kimi-Linear-48B-A3B-Instruct/snapshots/fd1de6347c9d3896f6df8edc529c68942bdd58f6" + +# Load model. +model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto") +processor = AutoProcessor.from_pretrained(MODEL_ID) + +# Configure the quantization algorithm and scheme. +# In this case, we: +# * quantize the weights to fp8 with channel-wise quantization +# * quantize the activations to fp8 with dynamic token activations +# NOTE: only datafree quantization is supported for Qwen3-VL-MoE currently +recipe = QuantizationModifier( + targets="Linear", + scheme="FP8_DYNAMIC", + ignore=[ + "re:.*lm_head", + "re:.*gate$", + "re:.*self_attn$", + ], +) + +# Apply quantization. +oneshot(model=model, recipe=recipe) + +print("========== SAMPLE GENERATION ==============") +dispatch_for_generation(model) +input_ids = tokenizer("Hello my name is", return_tensors="pt").input_ids.to( + model.device +) +output = model.generate(input_ids, max_new_tokens=20) +print(tokenizer.decode(output[0])) +print("==========================================") + +# Save to disk in compressed-tensors format. +SAVE_DIR = "/raid/engine/hub_cache/Kimi-Linear-48B-A3B-Instruct" + "-FP8-DYNAMIC" +model.save_pretrained(SAVE_DIR) +processor.save_pretrained(SAVE_DIR) From b2a3a3d029037366e58a0c8ad1d3728b705f539c Mon Sep 17 00:00:00 2001 From: Dipika Sikka Date: Fri, 31 Oct 2025 20:58:37 +0000 Subject: [PATCH 2/2] update --- examples/quantization_w8a8_fp8/kimi_linear_fp8.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/examples/quantization_w8a8_fp8/kimi_linear_fp8.py b/examples/quantization_w8a8_fp8/kimi_linear_fp8.py index 15b73d34a..2df3ad2f1 100644 --- a/examples/quantization_w8a8_fp8/kimi_linear_fp8.py +++ b/examples/quantization_w8a8_fp8/kimi_linear_fp8.py @@ -1,4 +1,4 @@ -from transformers import AutoProcessor, AutoModelForCausalLM +from transformers import AutoTokenizer, AutoModelForCausalLM from llmcompressor import oneshot from llmcompressor.modifiers.quantization import QuantizationModifier @@ -7,8 +7,8 @@ MODEL_ID = "//proving-grounds/engine/hub_cache/models--moonshotai--Kimi-Linear-48B-A3B-Instruct/snapshots/fd1de6347c9d3896f6df8edc529c68942bdd58f6" # Load model. -model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto") -processor = AutoProcessor.from_pretrained(MODEL_ID) +model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto", trust_remote_code=True) +tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True) # Configure the quantization algorithm and scheme. # In this case, we: @@ -28,6 +28,7 @@ # Apply quantization. oneshot(model=model, recipe=recipe) +""" print("========== SAMPLE GENERATION ==============") dispatch_for_generation(model) input_ids = tokenizer("Hello my name is", return_tensors="pt").input_ids.to( @@ -36,8 +37,9 @@ output = model.generate(input_ids, max_new_tokens=20) print(tokenizer.decode(output[0])) print("==========================================") +""" # Save to disk in compressed-tensors format. -SAVE_DIR = "/raid/engine/hub_cache/Kimi-Linear-48B-A3B-Instruct" + "-FP8-DYNAMIC" +SAVE_DIR = "/proving-grounds/engine/hub_cache/Kimi-Linear-48B-A3B-Instruct" + "-FP8-DYNAMIC" model.save_pretrained(SAVE_DIR) -processor.save_pretrained(SAVE_DIR) +tokenizer.save_pretrained(SAVE_DIR)