From ed7f39dadf8ced310d696ffdd7aaa42604bcdb31 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 Date: Thu, 2 Jul 2026 09:43:08 +0800 Subject: [PATCH 1/4] update swa --- .../models/paddleformers/base_fleet.py | 218 +++++++++++------- 1 file changed, 141 insertions(+), 77 deletions(-) diff --git a/fastdeploy/model_executor/models/paddleformers/base_fleet.py b/fastdeploy/model_executor/models/paddleformers/base_fleet.py index d3c22b11a2f..77a1fa2dec0 100644 --- a/fastdeploy/model_executor/models/paddleformers/base_fleet.py +++ b/fastdeploy/model_executor/models/paddleformers/base_fleet.py @@ -33,8 +33,6 @@ from paddlefleet.models.gpt.lm_head import GPTLMHead from paddlefleet.transformer.layer import FleetLayer from paddlefleet.transformer.transformer_config import TransformerConfig - from paddleformers.transformers import AutoConfig - from paddleformers.transformers.auto.modeling import AutoModelForCausalLM from paddleformers.utils.log import logger from fastdeploy.model_executor.forward_meta import ForwardMeta # noqa: F401 @@ -65,6 +63,8 @@ def __init__( hidden_size_per_attention_head: int, hidden_size_per_partition: int, layer_id: int, + window_attn_skip_freq=None, + sliding_window: int = 0, ): """ Initialize FastDeployAttention. @@ -86,6 +86,8 @@ def __init__( self.hidden_size_per_attention_head = hidden_size_per_attention_head self.hidden_size_per_partition = hidden_size_per_partition self.layer_id = layer_id + self.window_attn_skip_freq = window_attn_skip_freq + self.sliding_window = sliding_window def forward( self, @@ -167,85 +169,117 @@ def squeeze_to_3d(t: paddle.Tensor, name: str) -> paddle.Tensor: need_do_prefill = forward_meta.max_len_tensor_cpu[1] > 0 need_do_decode = forward_meta.max_len_tensor_cpu[2] > 0 - # MLA mode: pass q, k, v, compressed_kv, k_pe separately - # Reference: deepseek_v3.py line 389 - # - # Note: - # - Prefill (flash_attn_func): expects 3D tensors [seq, heads, dim] - # - Decode (multi_head_latent_attention): expects 2D tensors [seq, heads*dim] - # So we need to flatten q for decode phase only - - # Process compressed_kv and k_pe - assert kv_compressed is not None, "kv_compressed must be provided when use" compressed_kv = kv_compressed.squeeze(0) - k_pos_emb = k_pos_emb.squeeze(0) - - output = None - fmqa_out = None - if need_do_prefill: - # Prefill: keep 3D tensors for flash_attn_func - output = self.fd_attention.forward( - q=q, - k=k, - v=v, - qkv=None, - compressed_kv=compressed_kv, - k_pe=k_pos_emb, - forward_meta=forward_meta, + k_pos_emb_sq = k_pos_emb.squeeze(0) + + if self.window_attn_skip_freq is not None and self.window_attn_skip_freq[self.layer_id] == 1: + kv_lora_rank = self.config.kv_lora_rank + + q_input = squeeze_to_3d(q_absorbed, "q_absorbed") if q_absorbed.ndim == 4 else q_absorbed + num_attention_heads_tp = q_input.shape[1] + + """DSA sliding-window attention path, mirroring DeepseekV3MLAAttention.forward_swa_static.""" + from fastdeploy.model_executor.layers.attention import ( + DSAAttentionBackend, ) - output.reshape_([output.shape[0], output.shape[1] * output.shape[2]]) - - if need_do_decode: - # Decode: use absorbed q for multi_head_latent_attention C++ kernel - # q_absorbed: [s, heads, kv_lora_rank + qk_rope_head_dim] (after squeeze_to_3d) - # C++ kernel expects: [token_num, heads * (kv_lora_rank + qk_rope_head_dim)] - q_abs = squeeze_to_3d(q_absorbed, "q_absorbed") if q_absorbed.ndim == 4 else q_absorbed - seq_len = int(q_abs.shape[0]) - q_input = q_abs.reshape([seq_len, -1]) - - fmqa_out = self.fd_attention.forward( - q=q_input, - k=None, - v=None, - qkv=None, + from fastdeploy.model_executor.models.deepseek_v3 import ( + get_swa_indexer_top_k, + ) + + indexer_top_k = paddle.full([q_input.shape[0], 1, self.sliding_window[0]], -1, dtype="int32") + get_swa_indexer_top_k( + indexer_top_k, + forward_meta.block_tables, + forward_meta.cu_seqlens_q, + forward_meta.seq_lens_encoder, + forward_meta.seq_lens_decoder, + forward_meta.batch_id_per_token, + ) + fmqa_out = DSAAttentionBackend.forward_static( + q=q_input.contiguous(), + indexer_topk=indexer_top_k, compressed_kv=compressed_kv, - k_pe=k_pos_emb, + k_pe=k_pos_emb_sq, + latent_cache=forward_meta.caches[self.layer_id], forward_meta=forward_meta, + attn_softmax_scale=self.softmax_scale, ) - # V de-absorption: kernel output [token, heads * kv_lora_rank] - # -> [heads, token, kv_lora_rank] @ wv_b [heads, kv_lora_rank, v_head_dim] - # -> [token, heads * v_head_dim] - kv_lora_rank = self.config.kv_lora_rank - v_head_dim = self.config.v_head_dim - num_heads = fmqa_out.shape[-1] // kv_lora_rank - fmqa_out = fmqa_out.reshape([-1, num_heads, kv_lora_rank]).transpose([1, 0, 2]) + fmqa_out = fmqa_out.reshape_([-1, num_attention_heads_tp, kv_lora_rank]).transpose([1, 0, 2]) fmqa_out = paddle.bmm(fmqa_out, v_b_proj_weight) - fmqa_out = fmqa_out.transpose([1, 0, 2]).reshape([-1, num_heads * v_head_dim]) - # Merge prefill and decode outputs if both are present + output = fmqa_out.transpose([1, 0, 2]).reshape( + [-1, num_attention_heads_tp * self.config.v_head_dim] + ) + + else: + output = None + fmqa_out = None if need_do_prefill: - try: - from fastdeploy.model_executor.ops.gpu import ( - merge_prefill_decode_output, - ) - - merge_prefill_decode_output( - output, - fmqa_out, - forward_meta.seq_lens_encoder, - forward_meta.seq_lens_decoder, - forward_meta.seq_lens_this_time, - forward_meta.cu_seqlens_q, - num_heads, - v_head_dim, - 1, - ) - except (ImportError, AttributeError): - logger.warning("merge_prefill_decode_output not available, using decode output only") + # Prefill: keep 3D tensors for flash_attn_func + output = self.fd_attention.forward( + q=q, + k=k, + v=v, + qkv=None, + compressed_kv=compressed_kv, + k_pe=k_pos_emb_sq, + forward_meta=forward_meta, + ) + output.reshape_([output.shape[0], output.shape[1] * output.shape[2]]) + + if need_do_decode: + # Decode: use absorbed q for multi_head_latent_attention C++ kernel + # q_absorbed: [s, heads, kv_lora_rank + qk_rope_head_dim] (after squeeze_to_3d) + # C++ kernel expects: [token_num, heads * (kv_lora_rank + qk_rope_head_dim)] + q_abs = squeeze_to_3d(q_absorbed, "q_absorbed") if q_absorbed.ndim == 4 else q_absorbed + seq_len = int(q_abs.shape[0]) + q_input = q_abs.reshape([seq_len, -1]) + + fmqa_out = self.fd_attention.forward( + q=q_input, + k=None, + v=None, + qkv=None, + compressed_kv=compressed_kv, + k_pe=k_pos_emb_sq, + forward_meta=forward_meta, + ) + + # V de-absorption: kernel output [token, heads * kv_lora_rank] + # -> [heads, token, kv_lora_rank] @ wv_b [heads, kv_lora_rank, v_head_dim] + # -> [token, heads * v_head_dim] + kv_lora_rank = self.config.kv_lora_rank + v_head_dim = self.config.v_head_dim + num_heads = fmqa_out.shape[-1] // kv_lora_rank + fmqa_out = fmqa_out.reshape([-1, num_heads, kv_lora_rank]).transpose([1, 0, 2]) + fmqa_out = paddle.bmm(fmqa_out, v_b_proj_weight) + fmqa_out = fmqa_out.transpose([1, 0, 2]).reshape([-1, num_heads * v_head_dim]) + # Merge prefill and decode outputs if both are present + if need_do_prefill: + try: + from fastdeploy.model_executor.ops.gpu import ( + merge_prefill_decode_output, + ) + + merge_prefill_decode_output( + output, + fmqa_out, + forward_meta.seq_lens_encoder, + forward_meta.seq_lens_decoder, + forward_meta.seq_lens_this_time, + forward_meta.cu_seqlens_q, + num_heads, + v_head_dim, + 1, + ) + except (ImportError, AttributeError): + logger.warning( + "merge_prefill_decode_output not available, using decode output only" + ) + output = fmqa_out + else: output = fmqa_out - else: - output = fmqa_out else: # Standard mode: concatenate QKV seq_len = int(q.shape[0]) @@ -286,7 +320,21 @@ def __init__(self, fd_config: "FDConfig", **kwargs): logger.info("Initializing PaddleFormers backend.") self.fd_config = fd_config # FastDeploy's top-level FDConfig self.model_config = fd_config.model_config # FastDeploy's ModelConfig - self.paddleformers_config = AutoConfig.from_pretrained(self.model_config.model) + if True: + from ernie5.pretrain import Ernie5V2Config + from paddleformers.transformers.configuration_utils import ( + PretrainedConfig, + ) + + _config_dict, _ = PretrainedConfig.get_config_dict( + self.model_config.model, _configuration_file="model_config.json" + ) + self.paddleformers_config = Ernie5V2Config.from_dict(_config_dict) + self.paddleformers_config.moe_dequant_input = True + else: + from paddleformers.transformers import AutoConfig + + self.paddleformers_config = AutoConfig.from_pretrained(self.model_config.model) # Assign parallel config from fd_config.parallel_config to paddleformers_config parallel_config = fd_config.parallel_config @@ -306,6 +354,7 @@ def __init__(self, fd_config: "FDConfig", **kwargs): self.paddleformers_config.use_cpu_initialization = True self.paddleformers_config.perform_initialization = False self.paddleformers_config.gated_attention = getattr(self.paddleformers_config, "use_gated_attn", False) + self.paddleformers_config.moe_layer_interval = getattr(self.paddleformers_config, "moe_layer_freq", 1) if getattr(self.paddleformers_config, "multi_latent_attention", False): self.paddleformers_config.qk_head_dim = ( self.paddleformers_config.qk_rope_head_dim + self.paddleformers_config.qk_nope_head_dim @@ -334,11 +383,24 @@ def __init__(self, fd_config: "FDConfig", **kwargs): "load_via_cpu": True, "load_checkpoint_format": "flex_checkpoint", } - # Set random seed before model construction for reproducibility - self.model = AutoModelForCausalLM.from_pretrained( - self.model_config.model, - **model_load_kwargs, - ) + if True: + from fleet_bridge import AutoModelForCausalLM + + self.model = AutoModelForCausalLM.from_pretrained( + self.model_config.model, + config=self.paddleformers_config, + dtype=self.model_config.dtype, + ) + else: + from paddleformers.transformers.auto.modeling import ( + AutoModelForCausalLM, + ) + + # Set random seed before model construction for reproducibility + self.model = AutoModelForCausalLM.from_pretrained( + self.model_config.model, + **model_load_kwargs, + ) self.model.eval() # Patch PaddleFleet core_attention with FastDeploy attention @@ -721,6 +783,8 @@ def patch_paddlefleet_core_attention( hidden_size_per_attention_head=hidden_size_per_attention_head, hidden_size_per_partition=hidden_size_per_partition, layer_id=fd_layer_id, + window_attn_skip_freq=getattr(fd_config.model_config, "window_attn_skip_freq", None), + sliding_window=getattr(fd_config.model_config, "sliding_window", 0), ) # Replace core_attention object From bf0419a7fb7d01ee9136c534271339a5849724f6 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 Date: Thu, 2 Jul 2026 10:01:07 +0800 Subject: [PATCH 2/4] updtae test --- .../models/paddleformers/base_fleet.py | 6 +- .../test_fallback_fleet_model_coverge.py | 274 ++++++++++++++++++ 2 files changed, 278 insertions(+), 2 deletions(-) diff --git a/fastdeploy/model_executor/models/paddleformers/base_fleet.py b/fastdeploy/model_executor/models/paddleformers/base_fleet.py index 77a1fa2dec0..5982ad195d9 100644 --- a/fastdeploy/model_executor/models/paddleformers/base_fleet.py +++ b/fastdeploy/model_executor/models/paddleformers/base_fleet.py @@ -45,6 +45,8 @@ from fastdeploy.model_executor.layers.attention.attention import Attention + USE_ERNIE = False + class FastDeployAttention(FleetLayer): """ FastDeploy version of DotProductAttention, holding an internal FastDeploy Attention module. @@ -320,7 +322,7 @@ def __init__(self, fd_config: "FDConfig", **kwargs): logger.info("Initializing PaddleFormers backend.") self.fd_config = fd_config # FastDeploy's top-level FDConfig self.model_config = fd_config.model_config # FastDeploy's ModelConfig - if True: + if USE_ERNIE: from ernie5.pretrain import Ernie5V2Config from paddleformers.transformers.configuration_utils import ( PretrainedConfig, @@ -383,7 +385,7 @@ def __init__(self, fd_config: "FDConfig", **kwargs): "load_via_cpu": True, "load_checkpoint_format": "flex_checkpoint", } - if True: + if USE_ERNIE: from fleet_bridge import AutoModelForCausalLM self.model = AutoModelForCausalLM.from_pretrained( diff --git a/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py b/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py index a9f5c35c46e..0da4933e647 100644 --- a/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py +++ b/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py @@ -16,6 +16,7 @@ Covers: - FastDeployAttention.forward MLA branch (lines 166-249) +- FastDeployAttention.forward MLA DSA sliding-window attention path (lines 176-213) - FastDeployAttention.forward edge cases (squeeze_to_3d errors, scale restore) - patch_paddlefleet_core_attention error branches (lines 633-705) - PaddleFleetModelBase.forward zero-size & fallback branches (lines 530-561) @@ -1194,6 +1195,279 @@ def test_init_mla_model_computes_qk_head_dim(self): # ============================================================================ +# ============================================================================ +# Tests for FastDeployAttention.forward MLA DSA sliding-window attention path +# (lines 176-213 of base_fleet.py) +# ============================================================================ + + +def _create_mla_swa_attention(kv_lora_rank=4, v_head_dim=2, num_heads=2, layer_id=0, sliding_window=32): + """Create a FastDeployAttention with window_attn_skip_freq set so the SWA branch is taken.""" + mock_config = MagicMock() + mock_config.multi_latent_attention = True + mock_config.kv_lora_rank = kv_lora_rank + mock_config.v_head_dim = v_head_dim + + mock_fd_attention = MagicMock() + del mock_fd_attention.scale + + # window_attn_skip_freq[layer_id] == 1 triggers the SWA branch + window_attn_skip_freq = [1] * (layer_id + 1) + + with patch.object(FleetLayer, "__init__", lambda self, config: None): + attn = FastDeployAttention( + config=mock_config, + fd_attention=mock_fd_attention, + num_attention_heads=num_heads, + num_key_value_heads=num_heads, + softmax_scale=0.125, + hidden_size_per_attention_head=kv_lora_rank, + hidden_size_per_partition=num_heads * kv_lora_rank, + layer_id=layer_id, + window_attn_skip_freq=window_attn_skip_freq, + sliding_window=[sliding_window], + ) + attn.config = mock_config + return attn, mock_fd_attention + + +class TestFastDeployAttentionMLASWA: + """Test FastDeployAttention.forward DSA sliding-window attention branch (lines 176-213).""" + + def _make_forward_meta(self, seq_len, sliding_window, layer_id=0): + forward_meta = MagicMock() + # Both prefill and decode non-zero so is_mla check passes; + # the SWA branch exits early before the prefill/decode split. + forward_meta.max_len_tensor_cpu = [0, seq_len, 0] + forward_meta.block_tables = MagicMock() + forward_meta.cu_seqlens_q = MagicMock() + forward_meta.seq_lens_encoder = MagicMock() + forward_meta.seq_lens_decoder = MagicMock() + forward_meta.batch_id_per_token = MagicMock() + forward_meta.caches = {layer_id: MagicMock()} + return forward_meta + + def test_swa_branch_3d_q_absorbed(self): + """Lines 176-213: window_attn_skip_freq[layer_id]==1, q_absorbed 3D -> DSA path, + verifies reshape/bmm/transpose output shape [seq, heads*v_head_dim].""" + kv_lora_rank, v_head_dim, num_heads = 4, 2, 2 + seq_len = 3 + layer_id = 0 + sliding_window = 8 + attn, _ = _create_mla_swa_attention(kv_lora_rank, v_head_dim, num_heads, layer_id, sliding_window) + + forward_meta = self._make_forward_meta(seq_len, sliding_window, layer_id) + attn.config.forward_meta = forward_meta + + query = paddle.randn([seq_len, num_heads, kv_lora_rank]) + key = paddle.randn([seq_len, num_heads, kv_lora_rank]) + value = paddle.randn([seq_len, num_heads, kv_lora_rank]) + kv_compressed = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + k_pos_emb = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + # 3D q_absorbed: [seq, heads, kv_lora_rank] + q_absorbed = paddle.randn([seq_len, num_heads, kv_lora_rank]) + v_b_proj_weight = paddle.randn([num_heads, kv_lora_rank, v_head_dim]) + + # DSAAttentionBackend.forward_static returns [seq, heads * kv_lora_rank] + dsa_out_flat = paddle.randn([seq_len, num_heads * kv_lora_rank]) + + mock_dsa = MagicMock() + mock_dsa.forward_static = MagicMock(return_value=dsa_out_flat) + mock_get_swa_indexer = MagicMock() + + with patch.dict( + "sys.modules", + { + "fastdeploy.model_executor.layers.attention": MagicMock(DSAAttentionBackend=mock_dsa), + "fastdeploy.model_executor.models.deepseek_v3": MagicMock(get_swa_indexer_top_k=mock_get_swa_indexer), + }, + ): + result = attn.forward( + query=query, + key=key, + value=value, + attention_mask=None, + kv_compressed=kv_compressed, + k_pos_emb=k_pos_emb, + q_absorbed=q_absorbed, + v_b_proj_weight=v_b_proj_weight, + ) + + assert result is not None + # output shape after unsqueeze(0) in MLA return: [1, seq, heads*v_head_dim] + assert result.shape[0] == 1 + assert result.shape[1] == seq_len + assert result.shape[2] == num_heads * v_head_dim + mock_dsa.forward_static.assert_called_once() + mock_get_swa_indexer.assert_called_once() + + def test_swa_branch_4d_q_absorbed_squeeze(self): + """Line 179: q_absorbed 4D (batch=1) -> squeeze_to_3d called before DSA path.""" + kv_lora_rank, v_head_dim, num_heads = 4, 2, 2 + seq_len = 2 + layer_id = 0 + sliding_window = 8 + attn, _ = _create_mla_swa_attention(kv_lora_rank, v_head_dim, num_heads, layer_id, sliding_window) + + forward_meta = self._make_forward_meta(seq_len, sliding_window, layer_id) + attn.config.forward_meta = forward_meta + + query = paddle.randn([seq_len, num_heads, kv_lora_rank]) + key = paddle.randn([seq_len, num_heads, kv_lora_rank]) + value = paddle.randn([seq_len, num_heads, kv_lora_rank]) + kv_compressed = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + k_pos_emb = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + # 4D q_absorbed with batch=1 triggers the squeeze_to_3d branch on line 179 + q_absorbed = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + v_b_proj_weight = paddle.randn([num_heads, kv_lora_rank, v_head_dim]) + + dsa_out_flat = paddle.randn([seq_len, num_heads * kv_lora_rank]) + mock_dsa = MagicMock() + mock_dsa.forward_static = MagicMock(return_value=dsa_out_flat) + mock_get_swa_indexer = MagicMock() + + with patch.dict( + "sys.modules", + { + "fastdeploy.model_executor.layers.attention": MagicMock(DSAAttentionBackend=mock_dsa), + "fastdeploy.model_executor.models.deepseek_v3": MagicMock(get_swa_indexer_top_k=mock_get_swa_indexer), + }, + ): + result = attn.forward( + query=query, + key=key, + value=value, + attention_mask=None, + kv_compressed=kv_compressed, + k_pos_emb=k_pos_emb, + q_absorbed=q_absorbed, + v_b_proj_weight=v_b_proj_weight, + ) + + assert result is not None + assert result.shape[0] == 1 + assert result.shape[1] == seq_len + assert result.shape[2] == num_heads * v_head_dim + + def test_swa_branch_skip_freq_zero_bypasses_swa(self): + """window_attn_skip_freq[layer_id]==0 -> SWA branch NOT taken, falls through to normal MLA path.""" + kv_lora_rank, v_head_dim, num_heads = 4, 2, 2 + seq_len = 2 + layer_id = 0 + + mock_config = MagicMock() + mock_config.multi_latent_attention = True + mock_config.kv_lora_rank = kv_lora_rank + mock_config.v_head_dim = v_head_dim + + mock_fd_attention = MagicMock() + del mock_fd_attention.scale + + # skip_freq == 0 -> condition fails, normal path taken + window_attn_skip_freq = [0] + with patch.object(FleetLayer, "__init__", lambda self, config: None): + attn = FastDeployAttention( + config=mock_config, + fd_attention=mock_fd_attention, + num_attention_heads=num_heads, + num_key_value_heads=num_heads, + softmax_scale=0.125, + hidden_size_per_attention_head=kv_lora_rank, + hidden_size_per_partition=num_heads * kv_lora_rank, + layer_id=layer_id, + window_attn_skip_freq=window_attn_skip_freq, + sliding_window=[32], + ) + attn.config = mock_config + + forward_meta = MagicMock() + forward_meta.max_len_tensor_cpu = [0, seq_len, 0] + forward_meta.block_tables = MagicMock() + forward_meta.cu_seqlens_q = MagicMock() + forward_meta.seq_lens_encoder = MagicMock() + forward_meta.seq_lens_decoder = MagicMock() + forward_meta.batch_id_per_token = MagicMock() + forward_meta.caches = {layer_id: MagicMock()} + attn.config.forward_meta = forward_meta + + query = paddle.randn([seq_len, num_heads, kv_lora_rank]) + key = paddle.randn([seq_len, num_heads, kv_lora_rank]) + value = paddle.randn([seq_len, num_heads, kv_lora_rank]) + kv_compressed = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + k_pos_emb = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + + prefill_output = paddle.randn([seq_len, num_heads, kv_lora_rank]) + mock_fd_attention.forward.return_value = prefill_output + + result = attn.forward( + query=query, + key=key, + value=value, + attention_mask=None, + kv_compressed=kv_compressed, + k_pos_emb=k_pos_emb, + ) + # Normal MLA prefill path was used; DSA was never called + assert result is not None + mock_fd_attention.forward.assert_called_once() + + def test_swa_branch_indexer_shape(self): + """Line 190: indexer_top_k shape is [seq, 1, sliding_window[0]] filled with -1.""" + kv_lora_rank, v_head_dim, num_heads = 4, 2, 2 + seq_len = 5 + sliding_window = 16 + layer_id = 0 + attn, _ = _create_mla_swa_attention(kv_lora_rank, v_head_dim, num_heads, layer_id, sliding_window) + + forward_meta = self._make_forward_meta(seq_len, sliding_window, layer_id) + attn.config.forward_meta = forward_meta + + query = paddle.randn([seq_len, num_heads, kv_lora_rank]) + key = paddle.randn([seq_len, num_heads, kv_lora_rank]) + value = paddle.randn([seq_len, num_heads, kv_lora_rank]) + kv_compressed = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + k_pos_emb = paddle.randn([1, seq_len, num_heads, kv_lora_rank]) + q_absorbed = paddle.randn([seq_len, num_heads, kv_lora_rank]) + v_b_proj_weight = paddle.randn([num_heads, kv_lora_rank, v_head_dim]) + + captured = {} + + def fake_get_swa_indexer_top_k(indexer_top_k, *args, **kwargs): + captured["indexer_shape"] = list(indexer_top_k.shape) + captured["indexer_fill"] = int(indexer_top_k[0, 0, 0]) + + dsa_out_flat = paddle.randn([seq_len, num_heads * kv_lora_rank]) + mock_dsa = MagicMock() + mock_dsa.forward_static = MagicMock(return_value=dsa_out_flat) + + with patch.dict( + "sys.modules", + { + "fastdeploy.model_executor.layers.attention": MagicMock(DSAAttentionBackend=mock_dsa), + "fastdeploy.model_executor.models.deepseek_v3": MagicMock( + get_swa_indexer_top_k=fake_get_swa_indexer_top_k + ), + }, + ): + attn.forward( + query=query, + key=key, + value=value, + attention_mask=None, + kv_compressed=kv_compressed, + k_pos_emb=k_pos_emb, + q_absorbed=q_absorbed, + v_b_proj_weight=v_b_proj_weight, + ) + + assert captured["indexer_shape"] == [ + seq_len, + 1, + sliding_window, + ], f"Expected [{seq_len}, 1, {sliding_window}], got {captured['indexer_shape']}" + assert captured["indexer_fill"] == -1, f"Expected fill=-1, got {captured['indexer_fill']}" + + class TestTryResolvePaddlefleetImportError: """Test model_base.py line 203-209: paddlefleet not installed raises ImportError.""" From 4e0edcad1978a81165ffbe10dd0ad528447cf4cd Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 Date: Thu, 2 Jul 2026 15:03:28 +0800 Subject: [PATCH 3/4] clear test fix ci --- .../models/paddleformers/base_fleet.py | 3 +- .../test_fallback_fleet_model_coverge.py | 126 ++++++++++++++++-- 2 files changed, 115 insertions(+), 14 deletions(-) diff --git a/fastdeploy/model_executor/models/paddleformers/base_fleet.py b/fastdeploy/model_executor/models/paddleformers/base_fleet.py index 5982ad195d9..b33253195de 100644 --- a/fastdeploy/model_executor/models/paddleformers/base_fleet.py +++ b/fastdeploy/model_executor/models/paddleformers/base_fleet.py @@ -323,7 +323,6 @@ def __init__(self, fd_config: "FDConfig", **kwargs): self.fd_config = fd_config # FastDeploy's top-level FDConfig self.model_config = fd_config.model_config # FastDeploy's ModelConfig if USE_ERNIE: - from ernie5.pretrain import Ernie5V2Config from paddleformers.transformers.configuration_utils import ( PretrainedConfig, ) @@ -331,6 +330,8 @@ def __init__(self, fd_config: "FDConfig", **kwargs): _config_dict, _ = PretrainedConfig.get_config_dict( self.model_config.model, _configuration_file="model_config.json" ) + from ernie5.pretrain import Ernie5V2Config + self.paddleformers_config = Ernie5V2Config.from_dict(_config_dict) self.paddleformers_config.moe_dequant_input = True else: diff --git a/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py b/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py index 0da4933e647..82d1a77202a 100644 --- a/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py +++ b/tests/model_executor/fallback/test_fallback_fleet_model_coverge.py @@ -1126,8 +1126,58 @@ def _make_fd_config(self, multi_latent_attention=False): fd_config.graph_opt_config.use_cudagraph = False return fd_config + def _make_ernie_sys_mocks(self, mock_pf_config, mock_model): + """Return sys.modules patches for the USE_ERNIE=True code path in __init__. + + Because USE_ERNIE=True in this environment, AutoConfig / AutoModelForCausalLM + are imported locally (not module-level attrs), so we must mock them via + sys.modules rather than patch.object(bf_mod, ...). + """ + mock_ernie5_v2_config_cls = MagicMock() + mock_ernie5_v2_config_cls.from_dict = MagicMock(return_value=mock_pf_config) + + mock_pretrained_config = MagicMock() + mock_pretrained_config.get_config_dict = MagicMock(return_value=({}, None)) + + mock_fleet_bridge = MagicMock() + mock_fleet_bridge.AutoModelForCausalLM.from_pretrained = MagicMock(return_value=mock_model) + + return patch.dict( + "sys.modules", + { + "ernie5": MagicMock(), + "ernie5.pretrain": MagicMock(Ernie5V2Config=mock_ernie5_v2_config_cls), + "paddleformers.transformers.configuration_utils": MagicMock(PretrainedConfig=mock_pretrained_config), + "fleet_bridge": mock_fleet_bridge, + }, + ) + + def _make_paddleformers_sys_mocks(self, mock_pf_config, mock_model): + """Return sys.modules patches for the USE_ERNIE=False (default) code path. + + USE_ERNIE=False → AutoConfig.from_pretrained and paddleformers AutoModelForCausalLM + are used via local imports, so we mock them in sys.modules. + """ + mock_auto_config_cls = MagicMock() + mock_auto_config_cls.from_pretrained = MagicMock(return_value=mock_pf_config) + + mock_auto_model_cls = MagicMock() + mock_auto_model_cls.from_pretrained = MagicMock(return_value=mock_model) + + return ( + patch.dict( + "sys.modules", + { + "paddleformers.transformers": MagicMock(AutoConfig=mock_auto_config_cls), + "paddleformers.transformers.auto.modeling": MagicMock(AutoModelForCausalLM=mock_auto_model_cls), + }, + ), + mock_auto_config_cls, + mock_auto_model_cls, + ) + def test_init_standard_model(self): - """Lines 285-349: __init__ basic path (multi_latent_attention=False).""" + """Lines 285-349: __init__ basic path (USE_ERNIE=False, multi_latent_attention=False).""" import fastdeploy.model_executor.models.paddleformers.base_fleet as bf_mod fd_config = self._make_fd_config() @@ -1138,27 +1188,27 @@ def test_init_standard_model(self): mock_model = MagicMock() + sys_mocks, mock_auto_config_cls, mock_auto_model_cls = self._make_paddleformers_sys_mocks( + mock_pf_config, mock_model + ) + with ( - patch.object(bf_mod, "AutoConfig") as mock_ac, - patch.object(bf_mod, "AutoModelForCausalLM") as mock_am, + sys_mocks, patch.object(bf_mod, "patch_paddlefleet_core_attention", return_value=2), patch.object(PaddleFleetModelBase, "_init_paddlefleet_parallel_state"), patch.object(PaddleFleetModelBase, "_sync_config_from_text_config"), patch.object(paddle.nn.Layer, "__init__", lambda self, *a, **kw: None), ): - - mock_ac.from_pretrained.return_value = mock_pf_config - mock_am.from_pretrained.return_value = mock_model - model = object.__new__(PaddleFleetModelBase) PaddleFleetModelBase.__init__(model, fd_config) assert model.fd_config is fd_config assert model.paddleformers_config is mock_pf_config + mock_auto_config_cls.from_pretrained.assert_called_once_with(fd_config.model_config.model) mock_model.eval.assert_called_once() def test_init_mla_model_computes_qk_head_dim(self): - """Lines 309-312: multi_latent_attention=True → qk_head_dim computed from rope+nope.""" + """Lines 309-312: USE_ERNIE=False, multi_latent_attention=True → qk_head_dim = rope+nope.""" import fastdeploy.model_executor.models.paddleformers.base_fleet as bf_mod fd_config = self._make_fd_config() @@ -1171,22 +1221,72 @@ def test_init_mla_model_computes_qk_head_dim(self): mock_model = MagicMock() + sys_mocks, _, _ = self._make_paddleformers_sys_mocks(mock_pf_config, mock_model) + with ( - patch.object(bf_mod, "AutoConfig") as mock_ac, - patch.object(bf_mod, "AutoModelForCausalLM") as mock_am, + sys_mocks, patch.object(bf_mod, "patch_paddlefleet_core_attention", return_value=0), patch.object(PaddleFleetModelBase, "_init_paddlefleet_parallel_state"), patch.object(PaddleFleetModelBase, "_sync_config_from_text_config"), patch.object(paddle.nn.Layer, "__init__", lambda self, *a, **kw: None), ): + model = object.__new__(PaddleFleetModelBase) + PaddleFleetModelBase.__init__(model, fd_config) + + assert mock_pf_config.qk_head_dim == 64 + 128 + + def test_init_use_ernie_true_standard_model(self): + """USE_ERNIE=True path: ernie5 + fleet_bridge are used instead of paddleformers.""" + import fastdeploy.model_executor.models.paddleformers.base_fleet as bf_mod + + fd_config = self._make_fd_config() + + mock_pf_config = MagicMock() + mock_pf_config.tensor_model_parallel_size = 1 + mock_pf_config.multi_latent_attention = False - mock_ac.from_pretrained.return_value = mock_pf_config - mock_am.from_pretrained.return_value = mock_model + mock_model = MagicMock() + with ( + self._make_ernie_sys_mocks(mock_pf_config, mock_model), + patch.object(bf_mod, "USE_ERNIE", True), + patch.object(bf_mod, "patch_paddlefleet_core_attention", return_value=2), + patch.object(PaddleFleetModelBase, "_init_paddlefleet_parallel_state"), + patch.object(PaddleFleetModelBase, "_sync_config_from_text_config"), + patch.object(paddle.nn.Layer, "__init__", lambda self, *a, **kw: None), + ): + model = object.__new__(PaddleFleetModelBase) + PaddleFleetModelBase.__init__(model, fd_config) + + assert model.fd_config is fd_config + assert model.paddleformers_config is mock_pf_config + mock_model.eval.assert_called_once() + + def test_init_use_ernie_true_mla(self): + """USE_ERNIE=True + multi_latent_attention=True → qk_head_dim = rope+nope, ernie5 path.""" + import fastdeploy.model_executor.models.paddleformers.base_fleet as bf_mod + + fd_config = self._make_fd_config() + + mock_pf_config = MagicMock() + mock_pf_config.tensor_model_parallel_size = 1 + mock_pf_config.multi_latent_attention = True + mock_pf_config.qk_rope_head_dim = 64 + mock_pf_config.qk_nope_head_dim = 128 + + mock_model = MagicMock() + + with ( + self._make_ernie_sys_mocks(mock_pf_config, mock_model), + patch.object(bf_mod, "USE_ERNIE", True), + patch.object(bf_mod, "patch_paddlefleet_core_attention", return_value=0), + patch.object(PaddleFleetModelBase, "_init_paddlefleet_parallel_state"), + patch.object(PaddleFleetModelBase, "_sync_config_from_text_config"), + patch.object(paddle.nn.Layer, "__init__", lambda self, *a, **kw: None), + ): model = object.__new__(PaddleFleetModelBase) PaddleFleetModelBase.__init__(model, fd_config) - # qk_head_dim should be set to rope+nope assert mock_pf_config.qk_head_dim == 64 + 128 From 4ed6cab3f3f5c9aeb0b032ce91cff9574508d3e7 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 Date: Mon, 6 Jul 2026 14:18:31 +0800 Subject: [PATCH 4/4] fix --- .../model_executor/models/paddleformers/base_fleet.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/fastdeploy/model_executor/models/paddleformers/base_fleet.py b/fastdeploy/model_executor/models/paddleformers/base_fleet.py index b33253195de..42d511fffb9 100644 --- a/fastdeploy/model_executor/models/paddleformers/base_fleet.py +++ b/fastdeploy/model_executor/models/paddleformers/base_fleet.py @@ -189,7 +189,13 @@ def squeeze_to_3d(t: paddle.Tensor, name: str) -> paddle.Tensor: get_swa_indexer_top_k, ) - indexer_top_k = paddle.full([q_input.shape[0], 1, self.sliding_window[0]], -1, dtype="int32") + window_size = ( + self.sliding_window[0] + if isinstance(self.sliding_window, (list, tuple)) + else self.sliding_window + ) + indexer_top_k = paddle.full([q_input.shape[0], 1, window_size], -1, dtype="int32") + get_swa_indexer_top_k( indexer_top_k, forward_meta.block_tables,