Fix BatchEncoding.to() for nested elements (huggingface#38985)

Author: eginhard

src/transformers/tokenization_utils_base.py

@@ -801,14 +801,13 @@ def to(self, device: Union[str, "torch.device"], *, non_blocking: bool = False)
             [`BatchEncoding`]: The same instance after modification.
         """
         requires_backends(self, ["torch"])
-        import torch
 
         # This check catches things like APEX blindly calling "to" on all inputs to a module
         # Otherwise it passes the casts down and casts the LongTensor containing the token idxs
         # into a HalfTensor
         if isinstance(device, str) or is_torch_device(device) or isinstance(device, int):
             self.data = {
-                k: v.to(device=device, non_blocking=non_blocking) if isinstance(v, torch.Tensor) else v
+                k: v.to(device=device, non_blocking=non_blocking) if hasattr(v, "to") and callable(v.to) else v
                 for k, v in self.data.items()
             }
         else: