femu/nand: let a read preempt an in-flight program or erase

README.md

@@ -356,6 +356,8 @@ op_pcent=0             # Over-provisioning withheld from the host, percent
 pls_per_lun=1          # Planes per LUN; above one, a line erases across planes
 nand_bad_blocks=0      # Blocks marked bad at init, reflected in available spare
 trim_lat_ns=0          # Latency charged per TRIM range
+pe_suspend=0           # Reads preempt a program or erase in flight on their LUN
+tsusp_ns=0             #   (program/erase suspend); overhead each such read pays, ns
 
 # Wear, disturb and retention (optional; all default off)
 read_reclaim_limit=0   # Reads a block may take before its line is refreshed
@@ -573,6 +575,8 @@ zns_blk_er_lat=0
 zns_cmd_addr_lat=0     # Channel bus phases (ns): command/address cycle,
 zns_pg_xfer_lat=0      #   page data transfer, status read. Any non-zero
 zns_status_lat=0       #   value adds a shared per-channel bus to the model
+zns_pe_suspend=0       # Reads preempt a program or erase in flight on their plane
+zns_tsusp_ns=0         #   (program/erase suspend); overhead each such read pays, ns
 zns_zrwa_size=0        # ZRWA window in LBAs (0 = ZRWA disabled)
 zns_zrwafg_size=0      # ZRWA flush granularity in LBAs
 zns_zrwa_num=0         # Zones that may hold a ZRWA at once

hw/femu/bbssd/ftl-geom.c

@@ -184,6 +184,8 @@ void ssd_init_params(struct ssdparams *spp, FemuCtrl *n)
     spp->tplrbsy = n->bb_params.tplrbsy;
     spp->tplebsy = n->bb_params.tplebsy;
     spp->trcbsy = n->bb_params.trcbsy;
+    spp->pe_suspend = n->bb_params.pe_suspend;
+    spp->tsusp_ns = n->bb_params.tsusp_ns;
     spp->trim_lat_ns = n->bb_params.trim_lat_ns;
 
     /* DRAM write buffer */

hw/femu/bbssd/ftl-media.c

@@ -102,6 +102,8 @@ void bb_nand_media_init(struct ssd *ssd)
     cfg.timing.tplrbsy_ns = spp->tplrbsy;
     cfg.timing.tplebsy_ns = spp->tplebsy;
     cfg.timing.trcbsy_ns = spp->trcbsy;
+    cfg.policy.pe_suspend = (spp->pe_suspend != 0);
+    cfg.timing.tsusp_ns = spp->tsusp_ns;
     cfg.timing.ecc_step_ns = spp->ecc_step_ns;
     cfg.timing.ecc_pe_per_tier = FEMU_ECC_PE_PER_TIER;
     cfg.timing.ecc_max_tiers = FEMU_ECC_MAX_TIERS;

hw/femu/bbssd/ftl.h

@@ -152,6 +152,8 @@ struct ssdparams {
     int tplebsy;      /* multi-plane erase inter-plane busy (ns); 0 = off */
     int trcbsy;       /* cache read busy (next-page array overlap), ns; 0 = off */
     int trim_lat_ns;  /* modeled cost per processed DSM/TRIM range, ns; 0 = off */
+    int pe_suspend;   /* reads preempt an in-flight program/erase on the LUN; 0 = off */
+    int tsusp_ns;     /* suspend overhead per preempting read (ns) */
 
     double gc_thres_pcent;
     int gc_thres_lines;

hw/femu/femu.c

@@ -1905,6 +1905,8 @@ static const Property femu_props[] = {
     DEFINE_PROP_INT64("zns_cmd_addr_lat", FemuCtrl, zns_params.zns_cmd_addr_lat, 0),
     DEFINE_PROP_INT64("zns_pg_xfer_lat", FemuCtrl, zns_params.zns_pg_xfer_lat, 0),
     DEFINE_PROP_INT64("zns_status_lat", FemuCtrl, zns_params.zns_status_lat, 0),
+    DEFINE_PROP_INT32("zns_pe_suspend", FemuCtrl, zns_params.zns_pe_suspend, 0),
+    DEFINE_PROP_INT64("zns_tsusp_ns", FemuCtrl, zns_params.zns_tsusp_ns, 0),
     DEFINE_PROP_UINT32("zns_max_active", FemuCtrl, zns_params.zns_max_active, 0),
     DEFINE_PROP_UINT32("zns_max_open", FemuCtrl, zns_params.zns_max_open, 0),
     DEFINE_PROP_UINT32("zns_zd_ext_size", FemuCtrl, zns_params.zns_zd_ext_size, 0),
@@ -1956,6 +1958,8 @@ static const Property femu_props[] = {
     DEFINE_PROP_INT32("tplebsy", FemuCtrl, bb_params.tplebsy, 0),
     DEFINE_PROP_INT32("trcbsy", FemuCtrl, bb_params.trcbsy, 0),
     DEFINE_PROP_INT32("trim_lat_ns", FemuCtrl, bb_params.trim_lat_ns, 0),
+    DEFINE_PROP_INT32("pe_suspend", FemuCtrl, bb_params.pe_suspend, 0),
+    DEFINE_PROP_INT32("tsusp_ns", FemuCtrl, bb_params.tsusp_ns, 0),
     DEFINE_PROP_UINT32("nand_bad_blocks", FemuCtrl, nand_bad_blocks, 0),
     DEFINE_PROP_UINT32("op_pcent", FemuCtrl, op_pcent, 0),
     DEFINE_PROP_BOOL("debug_ftl", FemuCtrl, debug_ftl, false),

hw/femu/nand/nand-media.c

@@ -211,6 +211,30 @@ static uint64_t array_gate_start(NandMedia *m, const NandLoc *loc, uint64_t t)
     return s;
 }
 
+/*
+ * A read that preempted an in-flight P/E: push back every gated timeline that
+ * was busy at t by the read's occupancy, so the suspended operation finishes
+ * that much later. Timelines that were already idle at t are left alone.
+ */
+static void array_suspend_extend(NandMedia *m, const NandLoc *loc, uint64_t t,
+                                 uint64_t shift)
+{
+    if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY ||
+        m->cfg.policy.array_gate == NAND_GATE_LUN_AND_PLANE) {
+        uint64_t *lun = m->cfg.timeline->lun_avail(m->cfg.timeline_opaque, loc);
+        if (*lun > t) {
+            *lun += shift;
+        }
+    }
+    if (m->cfg.policy.array_gate == NAND_GATE_PLANE_ONLY ||
+        m->cfg.policy.array_gate == NAND_GATE_LUN_AND_PLANE) {
+        uint64_t *pl = m->cfg.timeline->plane_avail(m->cfg.timeline_opaque, loc);
+        if (*pl > t) {
+            *pl += shift;
+        }
+    }
+}
+
 static void array_commit(NandMedia *m, const NandLoc *loc, uint64_t done)
 {
     if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY ||
@@ -251,16 +275,33 @@ NandOpCompletion nand_media_op(NandMedia *m, const NandLoc *loc,
         if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY) {
             uint64_t *lun = m->cfg.timeline->lun_avail(m->cfg.timeline_opaque, loc);
             uint64_t old = __atomic_load_n(lun, __ATOMIC_RELAXED);
+            bool susp = m->cfg.policy.pe_suspend && op == NAND_MEDIA_READ;
             for (;;) {
-                uint64_t s = (t > old) ? t : old;
-                done = s + alat;
+                uint64_t s, avail;
+                if (susp && t < old) {
+                    /*
+                     * Program/erase suspend on the plain LUN gate: the read
+                     * starts after the suspend overhead instead of waiting out
+                     * the busy LUN, and whatever the LUN was doing resumes
+                     * after the read, so its completion slides by the read's
+                     * array time plus the overhead. Same arithmetic as the
+                     * staged path below.
+                     */
+                    s = t + m->cfg.timing.tsusp_ns;
+                    done = s + alat;
+                    avail = old + alat + m->cfg.timing.tsusp_ns;
+                } else {
+                    s = (t > old) ? t : old;
+                    done = s + alat;
+                    avail = done;
+                }
                 /*
                  * x86-64: inlines to `lock cmpxchg` (no libatomic call). On
                  * success `old` is unchanged and we stop; on failure the
                  * builtin writes the observed value back into `old` and we
                  * retry with it (recomputing done from what we actually saw).
                  */
-                if (__atomic_compare_exchange_n(lun, &old, done, false,
+                if (__atomic_compare_exchange_n(lun, &old, avail, false,
                                                 __ATOMIC_ACQ_REL,
                                                 __ATOMIC_RELAXED)) {
                     break;
@@ -275,8 +316,20 @@ NandOpCompletion nand_media_op(NandMedia *m, const NandLoc *loc,
         }
         {
             uint64_t s = array_gate_start(m, loc, t);
-            done = s + alat;
-            array_commit(m, loc, done);
+            if (m->cfg.policy.pe_suspend && op == NAND_MEDIA_READ && s > t) {
+                /*
+                 * Suspend on the plane / lun+plane gate (ZNS, OCSSD): the
+                 * read goes first, and every gated timeline that was busy
+                 * resumes its work after the read.
+                 */
+                uint64_t shift = alat + m->cfg.timing.tsusp_ns;
+                s = t + m->cfg.timing.tsusp_ns;
+                done = s + alat;
+                array_suspend_extend(m, loc, t, shift);
+            } else {
+                done = s + alat;
+                array_commit(m, loc, done);
+            }
         }
         if (m->cfg.timeline->unlock_lun) {
             m->cfg.timeline->unlock_lun(m->cfg.timeline_opaque, loc);

hw/femu/nand/nand-media.h

@@ -88,15 +88,17 @@ typedef struct NandMediaTiming {
     int32_t ecc_pe_per_tier;
     int32_t ecc_max_tiers;
     int32_t ecc_retention_per_tier_sec;
-    /* program/erase suspend overhead (ns) for an urgent read to preempt an in-flight P/E */
+    /* program/erase suspend overhead (ns) paid by a read that preempts an
+     * in-flight P/E on its LUN/plane (policy.pe_suspend); 0 = free suspend */
     int64_t tsusp_ns;
 } NandMediaTiming;
 
 typedef struct NandMediaPolicy {
     NandArrayGate   array_gate;
     NandChannelMode channel_mode;
     bool            cache_read;
-    bool            pe_suspend;   /* reads preempt an in-flight program/erase on the LUN */
+    bool            pe_suspend;   /* reads preempt an in-flight program/erase on the
+                                   * LUN/plane (all gates, staged or plain channel) */
     bool            ecc_on_read;
     bool            use_flat_timing;  /* true: scalar fields; false: table */
 } NandMediaPolicy;

hw/femu/nvme.h

@@ -1738,6 +1738,8 @@ typedef struct BbCtrlParams {
     int tplebsy;      /* multi-plane erase inter-plane busy (ns); 0 = off */
     int trcbsy;       /* cache read busy (next-page array overlap), ns; 0 = off */
     int trim_lat_ns;  /* modeled cost per processed DSM/TRIM range, ns; 0 = off */
+    int pe_suspend;   /* reads preempt an in-flight program/erase on the LUN; 0 = off */
+    int tsusp_ns;     /* suspend overhead per preempting read (ns) */
 } BbCtrlParams;
 
 typedef struct ZNSCtrlParams {
@@ -1759,6 +1761,9 @@ typedef struct ZNSCtrlParams {
     int64_t zns_cmd_addr_lat;
     int64_t zns_pg_xfer_lat;
     int64_t zns_status_lat;
+    /* program/erase suspend: reads preempt a busy plane (0 = off), overhead ns */
+    int32_t zns_pe_suspend;
+    int64_t zns_tsusp_ns;
     uint32_t zns_max_active;  /* max active zones (0 = unlimited) */
     uint32_t zns_max_open;    /* max open zones (0 = unlimited) */
     uint32_t zns_zd_ext_size; /* per-zone descriptor extension bytes (0 = none) */

hw/femu/tests/unit/test-nand-media.c

@@ -426,6 +426,98 @@ static void test_copyback(void)
     nand_media_destroy(&m);
 }
 
+/*
+ * Program/erase suspend. Off (the default) a read queues behind whatever the
+ * LUN or plane is doing; on, it starts after tsusp and the suspended
+ * operation resumes after it, so the LUN frees that much later. The same
+ * rule has to hold on the lock-free LUN gate (bbssd), the plane gate (ZNS)
+ * and the staged channel path, which is why each is checked.
+ */
+static void test_pe_suspend(void)
+{
+    NandMediaConfig cfg;
+    NandMedia m;
+    NandLoc a;
+    const uint64_t t0 = 1000000000ULL;
+    uint64_t lat;
+
+    printf("# program/erase suspend\n");
+    memset(&a, 0, sizeof(a));
+
+    /* lun-only gate, suspend off: the read waits out the program */
+    bb_config(&cfg);
+    nand_media_init(&m, &cfg);
+    nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns;
+    check("suspend off: read queues behind the program", lat, (40000 - 1000) + 10000);
+    nand_media_destroy(&m);
+
+    /* lun-only gate, suspend on: read after tsusp, program resumes after it */
+    bb_config(&cfg);
+    cfg.policy.pe_suspend = true;
+    cfg.timing.tsusp_ns = 5000;
+    nand_media_init(&m, &cfg);
+    nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns;
+    check("suspend on: read = tsusp + array", lat, 5000 + 10000);
+    /* the program now ends at t0 + 40000 + 10000 + 5000; a second program
+     * issued right after the read queues behind that */
+    lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns;
+    check("suspended program resumes after the read",
+          lat, (40000 + 10000 + 5000 - 1000) + 40000);
+    nand_media_destroy(&m);
+
+    /* an idle LUN pays no suspend overhead */
+    bb_config(&cfg);
+    cfg.policy.pe_suspend = true;
+    cfg.timing.tsusp_ns = 5000;
+    nand_media_init(&m, &cfg);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0).latency_ns;
+    check("suspend on, idle LUN: plain read", lat, 10000);
+    nand_media_destroy(&m);
+
+    /* only reads preempt: a program behind an erase still waits */
+    bb_config(&cfg);
+    cfg.policy.pe_suspend = true;
+    cfg.timing.tsusp_ns = 5000;
+    nand_media_init(&m, &cfg);
+    nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns;
+    check("suspend on: a program does not preempt", lat, (2000000 - 1000) + 40000);
+    nand_media_destroy(&m);
+
+    /* plane gate without a channel (ZNS default): a read preempts an erase */
+    zns_config(&cfg);
+    cfg.policy.channel_mode = NAND_CH_OFF;
+    cfg.policy.pe_suspend = true;
+    cfg.timing.tsusp_ns = 5000;
+    nand_media_init(&m, &cfg);
+    nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns;
+    check("plane gate: read preempts the erase", lat, 5000 + 65000);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns;
+    check("plane gate: erase resumes after the read",
+          lat, (2000000 + 65000 + 5000 - 1000) + 450000);
+    nand_media_destroy(&m);
+
+    /* staged channel: the existing suspend branch, now reachable */
+    bb_config(&cfg);
+    cfg.policy.channel_mode = NAND_CH_STAGED;
+    cfg.timing.cmd_addr_ns = 300;
+    cfg.timing.page_xfer_ns = 20000;
+    cfg.timing.status_ns = 100;
+    cfg.policy.pe_suspend = true;
+    cfg.timing.tsusp_ns = 5000;
+    nand_media_init(&m, &cfg);
+    nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0);
+    lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns;
+    /* command 300, then suspend 5000 + array 10000, then data-out 20000
+     * (the bus is idle: the erase moved no data) and status 100 */
+    check("staged: read = cmd + tsusp + array + xfer + status",
+          lat, 300 + 5000 + 10000 + 20000 + 100);
+    nand_media_destroy(&m);
+}
+
 int main(void)
 {
     /*
@@ -441,6 +533,7 @@ int main(void)
     test_plane_gate_with_channel();
     test_multiplane_erase();
     test_copyback();
+    test_pe_suspend();
     printf("1..%d\n", ntests);
     return failures ? 1 : 0;
 }

hw/femu/zns/zftl.c

@@ -118,6 +118,8 @@ void zns_nand_media_init(struct zns_ssd *zns)
     cfg.timing.cmd_addr_ns = zns->timing.cmd_addr_lat;
     cfg.timing.page_xfer_ns = zns->timing.pg_xfer_lat;
     cfg.timing.status_ns = zns->timing.status_lat;
+    cfg.policy.pe_suspend = zns->timing.pe_suspend;
+    cfg.timing.tsusp_ns = zns->timing.tsusp_ns;
     cfg.policy.channel_mode = (cfg.timing.cmd_addr_ns || cfg.timing.page_xfer_ns ||
                                cfg.timing.status_ns) ? NAND_CH_STAGED : NAND_CH_OFF;
     cfg.timeline = &zns_timeline_ops;

hw/femu/zns/zns.c

@@ -1857,6 +1857,9 @@ static void zns_init_params(FemuCtrl *n, NvmeNamespace *ns)
     id_zns->timing.cmd_addr_lat = n->zns_params.zns_cmd_addr_lat;
     id_zns->timing.pg_xfer_lat = n->zns_params.zns_pg_xfer_lat;
     id_zns->timing.status_lat = n->zns_params.zns_status_lat;
+    /* P/E suspend: default off, so the plane gate is bit-identical to before */
+    id_zns->timing.pe_suspend = (n->zns_params.zns_pe_suspend != 0);
+    id_zns->timing.tsusp_ns = n->zns_params.zns_tsusp_ns;
 
     /*
      * Optional write-fault injection. One write in N fails and takes its zone
@@ -1976,7 +1979,7 @@ static bool zns_check_params(FemuCtrl *n, NvmeNamespace *ns, Error **errp)
     }
     if (p->zns_pg_rd_lat < 0 || p->zns_pg_wr_lat < 0 || p->zns_blk_er_lat < 0 ||
         p->zns_cmd_addr_lat < 0 || p->zns_pg_xfer_lat < 0 ||
-        p->zns_status_lat < 0) {
+        p->zns_status_lat < 0 || p->zns_tsusp_ns < 0) {
         error_setg(errp, "zns NAND timing knobs must not be negative");
         return false;
     }

hw/femu/zns/zns.h

@@ -124,6 +124,9 @@ typedef struct SSDNandFlashTiming {
     uint64_t cmd_addr_lat;  /* command and address cycles */
     uint64_t pg_xfer_lat;   /* page data-in (program) / data-out (read) */
     uint64_t status_lat;    /* status read after a read or erase */
+    /* program/erase suspend: a read preempts a busy plane (0 = off) */
+    bool     pe_suspend;
+    uint64_t tsusp_ns;
 } SSDNandFlashTiming;
 
 struct zns_write_cache{