diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py
index 6e0c997c3eb0..01269bdddcc9 100644
--- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py
+++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py
@@ -184,7 +184,8 @@ def test_stable_diffusion_upscale_batch(self):
         assert image.shape[0] == 2
 
     def test_stable_diffusion_upscale_prompt_embeds(self):
-        sd_pipe = self.get_pipeline()
+        # `encode_prompt` is called with `torch_device` below, so the pipeline has to live there too.
+        sd_pipe = self.get_pipeline().to(torch_device)
 
         image = sd_pipe(**self.get_dummy_inputs()).images