fix adalora device mismatch issue (#583)

Author: younesbelkada

src/peft/tuners/lora.py

@@ -336,6 +336,8 @@ def _replace_module(self, parent_module, child_name, new_module, old_module):
         for name, module in new_module.named_modules():
             if "lora_" in name:
                 module.to(old_module.weight.device)
+            if "ranknum" in name:
+                module.to(old_module.weight.device)
 
     def __getattr__(self, name: str):
         """Forward missing attributes to the wrapped module."""