diff --git a/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py b/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py
index fd46ddd1b602..007d2b8da0cb 100644
--- a/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py
+++ b/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py
@@ -243,7 +243,9 @@ def _encode_vae_image(
         num_videos_per_prompt: int,
         do_classifier_free_guidance: bool,
     ):
-        image = image.to(device=device)
+        # The image comes out of `video_processor.preprocess()` in float32. When `needs_upcasting=False`, the VAE may not
+        # be in FP32 (for example, it could be in BF16) so we need to cast in this case.
+        image = image.to(device=device, dtype=self.vae.dtype)
         image_latents = self.vae.encode(image).latent_dist.mode()
 
         # duplicate image_latents for each generation per prompt, using mps friendly method