Fix lora_alpha in clip_benchmark (#701)

czczup · czczup · commit 6232f807e3a2 · 2024-11-23T23:40:29.000+08:00
Resolved inconsistency in lora_alpha configuration between training and evaluation that caused poor performance for LoRA fine-tuned models.
diff --git a/clip_benchmark/clip_benchmark/models/internvl_huggingface/modeling_internvl.py b/clip_benchmark/clip_benchmark/models/internvl_huggingface/modeling_internvl.py
@@ -206,9 +206,9 @@ def __init__(self, config: InternVLConfig):
         # self.post_init()
 
         if config.use_backbone_lora:
-            self.wrap_backbone_lora(r=config.use_backbone_lora)
+            self.wrap_backbone_lora(r=config.use_backbone_lora, lora_alpha=config.use_backbone_lora * 2)
         if config.use_qllama_lora:
-            self.wrap_qllama_lora(r=config.use_qllama_lora)
+            self.wrap_qllama_lora(r=config.use_qllama_lora, lora_alpha=config.use_qllama_lora * 2)
         if config.force_image_size:
             self.vision_model.resize_pos_embeddings(
                 old_size=config.vision_config.image_size,
