diff --git a/mmpretrain/models/multimodal/flamingo/flamingo.py b/mmpretrain/models/multimodal/flamingo/flamingo.py
index abdd03328f4..039f9ff35fb 100644
--- a/mmpretrain/models/multimodal/flamingo/flamingo.py
+++ b/mmpretrain/models/multimodal/flamingo/flamingo.py
@@ -96,6 +96,7 @@ def __init__(
                 map_location='cpu',
                 revise_keys=[(r'^backbone\.', '')],
             )
+            self.vision_encoder.is_init = True
 
         self.perceiver = PerceiverResampler(dim=self.vision_encoder.embed_dims)
 
diff --git a/mmpretrain/models/multimodal/llava/llava.py b/mmpretrain/models/multimodal/llava/llava.py
index 1c300fdcd05..103d81296f0 100644
--- a/mmpretrain/models/multimodal/llava/llava.py
+++ b/mmpretrain/models/multimodal/llava/llava.py
@@ -94,6 +94,7 @@ def __init__(self,
                 map_location='cpu',
                 revise_keys=[(r'^backbone\.', '')],
             )
+            vision_encoder.is_init = True
 
         # init language encoder related modules
         if load_lang_pretrained:
diff --git a/mmpretrain/models/multimodal/minigpt4/minigpt4.py b/mmpretrain/models/multimodal/minigpt4/minigpt4.py
index d23203603ec..4bbd5aaa299 100644
--- a/mmpretrain/models/multimodal/minigpt4/minigpt4.py
+++ b/mmpretrain/models/multimodal/minigpt4/minigpt4.py
@@ -79,6 +79,7 @@ def __init__(self,
         if vision_encoder_weight is not None:
             from mmengine.runner.checkpoint import load_checkpoint
             load_checkpoint(self.vision_encoder, vision_encoder_weight)
+            self.vision_encoder.is_init = True
         if freeze_vit:
             for name, param in self.ln_vision.named_parameters():
                 param.requires_grad = False
@@ -108,6 +109,9 @@ def __init__(self,
             state_dict = CheckpointLoader.load_checkpoint(
                 q_former_model_weight)['state_dict']
             self.load_state_dict(state_dict, strict=False)
+            # The ln_vision weights are also in the q-former checkpoint.
+            setattr(self.ln_vision, 'is_init', True)
+            setattr(self.q_former, 'is_init', True)
 
         if freeze_q_former:
             for name, param in self.q_former.named_parameters():
diff --git a/mmpretrain/models/multimodal/otter/otter.py b/mmpretrain/models/multimodal/otter/otter.py
index 2fed1a4d27c..5065c58c8f9 100644
--- a/mmpretrain/models/multimodal/otter/otter.py
+++ b/mmpretrain/models/multimodal/otter/otter.py
@@ -98,6 +98,7 @@ def __init__(
                 map_location='cpu',
                 revise_keys=[(r'^backbone\.', '')],
             )
+            self.vision_encoder.is_init = True
 
         self.perceiver = PerceiverResampler(dim=self.vision_encoder.embed_dims)
 
diff --git a/mmpretrain/models/utils/huggingface.py b/mmpretrain/models/utils/huggingface.py
index e527315b26e..a44d6daaf1c 100644
--- a/mmpretrain/models/utils/huggingface.py
+++ b/mmpretrain/models/utils/huggingface.py
@@ -86,7 +86,9 @@ def build(**kwargs):
                                   kwargs.pop('name_or_path'))
 
         if kwargs.pop('load_pretrained', True) and _load_hf_pretrained_model:
-            return cls.from_pretrained(name_or_path, **kwargs)
+            model = cls.from_pretrained(name_or_path, **kwargs)
+            setattr(model, 'is_init', True)
+            return model
         else:
             cfg = get_config(name_or_path, **kwargs)
             return from_config(cfg)