diff --git a/swift/arguments/tuner_args.py b/swift/arguments/tuner_args.py index b742cec56a..978c857bd8 100644 --- a/swift/arguments/tuner_args.py +++ b/swift/arguments/tuner_args.py @@ -52,7 +52,7 @@ class TunerArguments: modules_to_save (List[str]): List of modules to save. Default is an empty list. lora_rank (int): Rank for LoRA. Default is 8. - lora_alpha (int): Alpha value for LoRA. Default is 32. + lora_alpha (float): Alpha value for LoRA. Default is 32.0. lora_dropout (float): Dropout rate for LoRA. Default is 0.05. lora_bias (Literal['none', 'all']): The possible values are 'none' and 'all'. If set to 'all', all biases will be trainable. Default is 'none'. @@ -131,7 +131,7 @@ class TunerArguments: # lora lora_rank: int = 8 - lora_alpha: int = 32 + lora_alpha: float = 32.0 lora_dropout: float = 0.05 lora_bias: Literal['none', 'all'] = 'none' lora_dtype: Literal['float16', 'bfloat16', 'float32', None] = None diff --git a/swift/megatron/arguments/megatron_args.py b/swift/megatron/arguments/megatron_args.py index 673bad351d..d2438f97bf 100644 --- a/swift/megatron/arguments/megatron_args.py +++ b/swift/megatron/arguments/megatron_args.py @@ -463,7 +463,7 @@ class MegatronTunerMixin: # lora lora_rank: int = 8 - lora_alpha: int = 32 + lora_alpha: float = 32.0 lora_dropout: float = 0.05 lora_bias: Literal['none', 'all'] = 'none' lora_dtype: Literal['float16', 'bfloat16', 'float32', None] = None diff --git a/swift/tuners/lora_layers.py b/swift/tuners/lora_layers.py index 7f6b7a399e..54e42e14dc 100644 --- a/swift/tuners/lora_layers.py +++ b/swift/tuners/lora_layers.py @@ -517,7 +517,7 @@ def __init__( adapter_name: str, module_key: str, r: int, - lora_alpha: int, + lora_alpha: float, lora_dropout: float, merge_weights: bool, ): @@ -544,7 +544,7 @@ def __init__(self, module_key: str, base_layer: nn.Linear, r: int = 0, - lora_alpha: int = 1, + lora_alpha: float = 1.0, lora_dropout: float = 0., enable_lora: List[bool] = [False], fan_in_fan_out: bool = False, diff --git a/tests/megatron/test_opsd.py b/tests/megatron/test_opsd.py index 40932f8451..a337fbeb47 100644 --- a/tests/megatron/test_opsd.py +++ b/tests/megatron/test_opsd.py @@ -19,7 +19,7 @@ vllm_max_model_len=10240, tuner_type='lora', lora_rank=64, - lora_alpha=128, + lora_alpha=128.0, sleep_level=1, lmbda=1.0, beta=0.5, diff --git a/tests/train/test_opsd.py b/tests/train/test_opsd.py index 3aa2b93fee..43c4f6c99b 100644 --- a/tests/train/test_opsd.py +++ b/tests/train/test_opsd.py @@ -13,7 +13,7 @@ def train(): teacher_model='Qwen/Qwen3.5-4B', tuner_type='lora', lora_rank=64, - lora_alpha=128, + lora_alpha=128.0, target_modules=['all-linear'], use_vllm=True, vllm_mode='colocate',