diff --git a/.github/workflows/nv-v100-legacy.yml b/.github/workflows/nv-v100-legacy.yml
index 1a35f1f1..346e7f18 100644
--- a/.github/workflows/nv-v100-legacy.yml
+++ b/.github/workflows/nv-v100-legacy.yml
@@ -36,7 +36,7 @@ jobs:
       - name: Install dependencies
         run: |
           pip install git+https://github.com/microsoft/DeepSpeed.git
-          pip install git+https://github.com/huggingface/transformers.git@v4.42.4
+          pip install git+https://github.com/huggingface/transformers.git@v4.43.4
           pip install -U accelerate
           ds_report
 
diff --git a/mii/legacy/models/load_models.py b/mii/legacy/models/load_models.py
index cfbf455f..bbb7e37b 100644
--- a/mii/legacy/models/load_models.py
+++ b/mii/legacy/models/load_models.py
@@ -75,6 +75,8 @@ def load_models(model_config):
         if model_config.profile_model_time:
             engine.profile_model_time()
         if hasattr(inference_pipeline, "model"):
+            engine._parameters = inference_pipeline.model._parameters
+            engine.training = inference_pipeline.model.training
             inference_pipeline.model = engine
 
     elif model_config.enable_zero: