diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanBatchPrefillDecode.java b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanBatchPrefillDecode.java index a0fb1c1b..e80e757f 100644 --- a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanBatchPrefillDecode.java +++ b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanBatchPrefillDecode.java @@ -78,21 +78,28 @@ public TornadoExecutionPlan createExecutionPlan() { // ── Initialisation ──────────────────────────────────────────────────────── + /** + * Puts the read-only weights on the device. With CUDA graphs enabled the pass is also the + * capture, so it still runs every graph; otherwise it is a plain copy-in and no kernel runs. + */ // @formatter:off @Override public void forceCopyInReadOnlyData() { + if (!CUDA_GRAPHS) { + executionPlan.transferToDevice(); + return; + } + state.wrapX.clear(); state.positionHolder.init(0); state.wrapXBatch.clear(); state.batchStartPosHolder.init(0); for (int i = 0; i <= taskGraphLayout.logitsIdx(); i++) { - var g = executionPlan.withGraph(i) - .withGridScheduler(batchPrefillDecodeForwardPlan.getGridScheduler()); - if (CUDA_GRAPHS) { - g.withCUDAGraph(); - } - g.execute(); + executionPlan.withGraph(i) + .withGridScheduler(batchPrefillDecodeForwardPlan.getGridScheduler()) + .withCUDAGraph() + .execute(); } } // @formatter:on diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanPrefillDecode.java b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanPrefillDecode.java index 4ba9c834..73f9f47d 100644 --- a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanPrefillDecode.java +++ b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanPrefillDecode.java @@ -86,20 +86,26 @@ public TornadoExecutionPlan createExecutionPlan() { // ── Initialisation ──────────────────────────────────────────────────────── - /** Runs all graphs once to trigger FIRST_EXECUTION uploads and warm up CUDA graphs. */ + /** + * Puts the read-only weights on the device. With CUDA graphs enabled the pass is also the + * capture, so it still runs every graph; otherwise it is a plain copy-in and no kernel runs. + */ // @formatter:off @Override public void forceCopyInReadOnlyData() { + if (!CUDA_GRAPHS) { + executionPlan.transferToDevice(); + return; + } + state.wrapX.clear(); state.positionHolder.init(0); for (int i = 0; i <= taskGraphLayout.logitsIdx(); i++) { - var g = executionPlan.withGraph(i) - .withGridScheduler(prefillDecodeForwardPlan.getGridScheduler()); - if (CUDA_GRAPHS) { - g.withCUDAGraph(); - } - g.execute(); + executionPlan.withGraph(i) + .withGridScheduler(prefillDecodeForwardPlan.getGridScheduler()) + .withCUDAGraph() + .execute(); } } // @formatter:on diff --git a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanSingleToken.java b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanSingleToken.java index 5035d384..df8aa0ad 100644 --- a/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanSingleToken.java +++ b/src/main/java/org/beehive/gpullama3/tornadovm/TornadoVMMasterPlanSingleToken.java @@ -95,9 +95,18 @@ public FloatArray tornadoVMForwardDecode(int position) { } // @formatter:on + /** + * Puts the read-only weights on the device. With CUDA graphs enabled the pass is also the + * capture, so it still runs every graph; otherwise it is a plain copy-in and no kernel runs. + */ // @formatter:off @Override public void forceCopyInReadOnlyData() { + if (!CUDA_GRAPHS) { + executionPlan.transferToDevice(); + return; + } + state.wrapX.clear(); state.positionHolder.init(0);