diff --git a/README.md b/README.md index d3ea9c168e3..66d1454476e 100644 --- a/README.md +++ b/README.md @@ -356,6 +356,8 @@ op_pcent=0 # Over-provisioning withheld from the host, percent pls_per_lun=1 # Planes per LUN; above one, a line erases across planes nand_bad_blocks=0 # Blocks marked bad at init, reflected in available spare trim_lat_ns=0 # Latency charged per TRIM range +pe_suspend=0 # Reads preempt a program or erase in flight on their LUN +tsusp_ns=0 # (program/erase suspend); overhead each such read pays, ns # Wear, disturb and retention (optional; all default off) read_reclaim_limit=0 # Reads a block may take before its line is refreshed @@ -573,6 +575,8 @@ zns_blk_er_lat=0 zns_cmd_addr_lat=0 # Channel bus phases (ns): command/address cycle, zns_pg_xfer_lat=0 # page data transfer, status read. Any non-zero zns_status_lat=0 # value adds a shared per-channel bus to the model +zns_pe_suspend=0 # Reads preempt a program or erase in flight on their plane +zns_tsusp_ns=0 # (program/erase suspend); overhead each such read pays, ns zns_zrwa_size=0 # ZRWA window in LBAs (0 = ZRWA disabled) zns_zrwafg_size=0 # ZRWA flush granularity in LBAs zns_zrwa_num=0 # Zones that may hold a ZRWA at once diff --git a/hw/femu/bbssd/ftl-geom.c b/hw/femu/bbssd/ftl-geom.c index 18c06ff986b..38730b209f8 100644 --- a/hw/femu/bbssd/ftl-geom.c +++ b/hw/femu/bbssd/ftl-geom.c @@ -184,6 +184,8 @@ void ssd_init_params(struct ssdparams *spp, FemuCtrl *n) spp->tplrbsy = n->bb_params.tplrbsy; spp->tplebsy = n->bb_params.tplebsy; spp->trcbsy = n->bb_params.trcbsy; + spp->pe_suspend = n->bb_params.pe_suspend; + spp->tsusp_ns = n->bb_params.tsusp_ns; spp->trim_lat_ns = n->bb_params.trim_lat_ns; /* DRAM write buffer */ diff --git a/hw/femu/bbssd/ftl-media.c b/hw/femu/bbssd/ftl-media.c index f173f5b11c4..ea34638352e 100644 --- a/hw/femu/bbssd/ftl-media.c +++ b/hw/femu/bbssd/ftl-media.c @@ -102,6 +102,8 @@ void bb_nand_media_init(struct ssd *ssd) cfg.timing.tplrbsy_ns = spp->tplrbsy; cfg.timing.tplebsy_ns = spp->tplebsy; cfg.timing.trcbsy_ns = spp->trcbsy; + cfg.policy.pe_suspend = (spp->pe_suspend != 0); + cfg.timing.tsusp_ns = spp->tsusp_ns; cfg.timing.ecc_step_ns = spp->ecc_step_ns; cfg.timing.ecc_pe_per_tier = FEMU_ECC_PE_PER_TIER; cfg.timing.ecc_max_tiers = FEMU_ECC_MAX_TIERS; diff --git a/hw/femu/bbssd/ftl.h b/hw/femu/bbssd/ftl.h index 094873368e0..6edded852ca 100644 --- a/hw/femu/bbssd/ftl.h +++ b/hw/femu/bbssd/ftl.h @@ -152,6 +152,8 @@ struct ssdparams { int tplebsy; /* multi-plane erase inter-plane busy (ns); 0 = off */ int trcbsy; /* cache read busy (next-page array overlap), ns; 0 = off */ int trim_lat_ns; /* modeled cost per processed DSM/TRIM range, ns; 0 = off */ + int pe_suspend; /* reads preempt an in-flight program/erase on the LUN; 0 = off */ + int tsusp_ns; /* suspend overhead per preempting read (ns) */ double gc_thres_pcent; int gc_thres_lines; diff --git a/hw/femu/femu.c b/hw/femu/femu.c index edf4ca2666b..f089ad36835 100644 --- a/hw/femu/femu.c +++ b/hw/femu/femu.c @@ -1905,6 +1905,8 @@ static const Property femu_props[] = { DEFINE_PROP_INT64("zns_cmd_addr_lat", FemuCtrl, zns_params.zns_cmd_addr_lat, 0), DEFINE_PROP_INT64("zns_pg_xfer_lat", FemuCtrl, zns_params.zns_pg_xfer_lat, 0), DEFINE_PROP_INT64("zns_status_lat", FemuCtrl, zns_params.zns_status_lat, 0), + DEFINE_PROP_INT32("zns_pe_suspend", FemuCtrl, zns_params.zns_pe_suspend, 0), + DEFINE_PROP_INT64("zns_tsusp_ns", FemuCtrl, zns_params.zns_tsusp_ns, 0), DEFINE_PROP_UINT32("zns_max_active", FemuCtrl, zns_params.zns_max_active, 0), DEFINE_PROP_UINT32("zns_max_open", FemuCtrl, zns_params.zns_max_open, 0), DEFINE_PROP_UINT32("zns_zd_ext_size", FemuCtrl, zns_params.zns_zd_ext_size, 0), @@ -1956,6 +1958,8 @@ static const Property femu_props[] = { DEFINE_PROP_INT32("tplebsy", FemuCtrl, bb_params.tplebsy, 0), DEFINE_PROP_INT32("trcbsy", FemuCtrl, bb_params.trcbsy, 0), DEFINE_PROP_INT32("trim_lat_ns", FemuCtrl, bb_params.trim_lat_ns, 0), + DEFINE_PROP_INT32("pe_suspend", FemuCtrl, bb_params.pe_suspend, 0), + DEFINE_PROP_INT32("tsusp_ns", FemuCtrl, bb_params.tsusp_ns, 0), DEFINE_PROP_UINT32("nand_bad_blocks", FemuCtrl, nand_bad_blocks, 0), DEFINE_PROP_UINT32("op_pcent", FemuCtrl, op_pcent, 0), DEFINE_PROP_BOOL("debug_ftl", FemuCtrl, debug_ftl, false), diff --git a/hw/femu/nand/nand-media.c b/hw/femu/nand/nand-media.c index 790cc8625d4..9d774d8a908 100644 --- a/hw/femu/nand/nand-media.c +++ b/hw/femu/nand/nand-media.c @@ -211,6 +211,30 @@ static uint64_t array_gate_start(NandMedia *m, const NandLoc *loc, uint64_t t) return s; } +/* + * A read that preempted an in-flight P/E: push back every gated timeline that + * was busy at t by the read's occupancy, so the suspended operation finishes + * that much later. Timelines that were already idle at t are left alone. + */ +static void array_suspend_extend(NandMedia *m, const NandLoc *loc, uint64_t t, + uint64_t shift) +{ + if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY || + m->cfg.policy.array_gate == NAND_GATE_LUN_AND_PLANE) { + uint64_t *lun = m->cfg.timeline->lun_avail(m->cfg.timeline_opaque, loc); + if (*lun > t) { + *lun += shift; + } + } + if (m->cfg.policy.array_gate == NAND_GATE_PLANE_ONLY || + m->cfg.policy.array_gate == NAND_GATE_LUN_AND_PLANE) { + uint64_t *pl = m->cfg.timeline->plane_avail(m->cfg.timeline_opaque, loc); + if (*pl > t) { + *pl += shift; + } + } +} + static void array_commit(NandMedia *m, const NandLoc *loc, uint64_t done) { if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY || @@ -251,16 +275,33 @@ NandOpCompletion nand_media_op(NandMedia *m, const NandLoc *loc, if (m->cfg.policy.array_gate == NAND_GATE_LUN_ONLY) { uint64_t *lun = m->cfg.timeline->lun_avail(m->cfg.timeline_opaque, loc); uint64_t old = __atomic_load_n(lun, __ATOMIC_RELAXED); + bool susp = m->cfg.policy.pe_suspend && op == NAND_MEDIA_READ; for (;;) { - uint64_t s = (t > old) ? t : old; - done = s + alat; + uint64_t s, avail; + if (susp && t < old) { + /* + * Program/erase suspend on the plain LUN gate: the read + * starts after the suspend overhead instead of waiting out + * the busy LUN, and whatever the LUN was doing resumes + * after the read, so its completion slides by the read's + * array time plus the overhead. Same arithmetic as the + * staged path below. + */ + s = t + m->cfg.timing.tsusp_ns; + done = s + alat; + avail = old + alat + m->cfg.timing.tsusp_ns; + } else { + s = (t > old) ? t : old; + done = s + alat; + avail = done; + } /* * x86-64: inlines to `lock cmpxchg` (no libatomic call). On * success `old` is unchanged and we stop; on failure the * builtin writes the observed value back into `old` and we * retry with it (recomputing done from what we actually saw). */ - if (__atomic_compare_exchange_n(lun, &old, done, false, + if (__atomic_compare_exchange_n(lun, &old, avail, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { break; @@ -275,8 +316,20 @@ NandOpCompletion nand_media_op(NandMedia *m, const NandLoc *loc, } { uint64_t s = array_gate_start(m, loc, t); - done = s + alat; - array_commit(m, loc, done); + if (m->cfg.policy.pe_suspend && op == NAND_MEDIA_READ && s > t) { + /* + * Suspend on the plane / lun+plane gate (ZNS, OCSSD): the + * read goes first, and every gated timeline that was busy + * resumes its work after the read. + */ + uint64_t shift = alat + m->cfg.timing.tsusp_ns; + s = t + m->cfg.timing.tsusp_ns; + done = s + alat; + array_suspend_extend(m, loc, t, shift); + } else { + done = s + alat; + array_commit(m, loc, done); + } } if (m->cfg.timeline->unlock_lun) { m->cfg.timeline->unlock_lun(m->cfg.timeline_opaque, loc); diff --git a/hw/femu/nand/nand-media.h b/hw/femu/nand/nand-media.h index f878f790347..994cf5b1ccd 100644 --- a/hw/femu/nand/nand-media.h +++ b/hw/femu/nand/nand-media.h @@ -88,7 +88,8 @@ typedef struct NandMediaTiming { int32_t ecc_pe_per_tier; int32_t ecc_max_tiers; int32_t ecc_retention_per_tier_sec; - /* program/erase suspend overhead (ns) for an urgent read to preempt an in-flight P/E */ + /* program/erase suspend overhead (ns) paid by a read that preempts an + * in-flight P/E on its LUN/plane (policy.pe_suspend); 0 = free suspend */ int64_t tsusp_ns; } NandMediaTiming; @@ -96,7 +97,8 @@ typedef struct NandMediaPolicy { NandArrayGate array_gate; NandChannelMode channel_mode; bool cache_read; - bool pe_suspend; /* reads preempt an in-flight program/erase on the LUN */ + bool pe_suspend; /* reads preempt an in-flight program/erase on the + * LUN/plane (all gates, staged or plain channel) */ bool ecc_on_read; bool use_flat_timing; /* true: scalar fields; false: table */ } NandMediaPolicy; diff --git a/hw/femu/nvme.h b/hw/femu/nvme.h index 021efe77d23..50af0837c8b 100644 --- a/hw/femu/nvme.h +++ b/hw/femu/nvme.h @@ -1738,6 +1738,8 @@ typedef struct BbCtrlParams { int tplebsy; /* multi-plane erase inter-plane busy (ns); 0 = off */ int trcbsy; /* cache read busy (next-page array overlap), ns; 0 = off */ int trim_lat_ns; /* modeled cost per processed DSM/TRIM range, ns; 0 = off */ + int pe_suspend; /* reads preempt an in-flight program/erase on the LUN; 0 = off */ + int tsusp_ns; /* suspend overhead per preempting read (ns) */ } BbCtrlParams; typedef struct ZNSCtrlParams { @@ -1759,6 +1761,9 @@ typedef struct ZNSCtrlParams { int64_t zns_cmd_addr_lat; int64_t zns_pg_xfer_lat; int64_t zns_status_lat; + /* program/erase suspend: reads preempt a busy plane (0 = off), overhead ns */ + int32_t zns_pe_suspend; + int64_t zns_tsusp_ns; uint32_t zns_max_active; /* max active zones (0 = unlimited) */ uint32_t zns_max_open; /* max open zones (0 = unlimited) */ uint32_t zns_zd_ext_size; /* per-zone descriptor extension bytes (0 = none) */ diff --git a/hw/femu/tests/test-nand-media b/hw/femu/tests/test-nand-media new file mode 100755 index 00000000000..2e93ac44e07 Binary files /dev/null and b/hw/femu/tests/test-nand-media differ diff --git a/hw/femu/tests/unit/test-nand-media.c b/hw/femu/tests/unit/test-nand-media.c index c52c11f7877..fa7698b3a58 100644 --- a/hw/femu/tests/unit/test-nand-media.c +++ b/hw/femu/tests/unit/test-nand-media.c @@ -426,6 +426,98 @@ static void test_copyback(void) nand_media_destroy(&m); } +/* + * Program/erase suspend. Off (the default) a read queues behind whatever the + * LUN or plane is doing; on, it starts after tsusp and the suspended + * operation resumes after it, so the LUN frees that much later. The same + * rule has to hold on the lock-free LUN gate (bbssd), the plane gate (ZNS) + * and the staged channel path, which is why each is checked. + */ +static void test_pe_suspend(void) +{ + NandMediaConfig cfg; + NandMedia m; + NandLoc a; + const uint64_t t0 = 1000000000ULL; + uint64_t lat; + + printf("# program/erase suspend\n"); + memset(&a, 0, sizeof(a)); + + /* lun-only gate, suspend off: the read waits out the program */ + bb_config(&cfg); + nand_media_init(&m, &cfg); + nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0); + lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns; + check("suspend off: read queues behind the program", lat, (40000 - 1000) + 10000); + nand_media_destroy(&m); + + /* lun-only gate, suspend on: read after tsusp, program resumes after it */ + bb_config(&cfg); + cfg.policy.pe_suspend = true; + cfg.timing.tsusp_ns = 5000; + nand_media_init(&m, &cfg); + nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0); + lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns; + check("suspend on: read = tsusp + array", lat, 5000 + 10000); + /* the program now ends at t0 + 40000 + 10000 + 5000; a second program + * issued right after the read queues behind that */ + lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns; + check("suspended program resumes after the read", + lat, (40000 + 10000 + 5000 - 1000) + 40000); + nand_media_destroy(&m); + + /* an idle LUN pays no suspend overhead */ + bb_config(&cfg); + cfg.policy.pe_suspend = true; + cfg.timing.tsusp_ns = 5000; + nand_media_init(&m, &cfg); + lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0).latency_ns; + check("suspend on, idle LUN: plain read", lat, 10000); + nand_media_destroy(&m); + + /* only reads preempt: a program behind an erase still waits */ + bb_config(&cfg); + cfg.policy.pe_suspend = true; + cfg.timing.tsusp_ns = 5000; + nand_media_init(&m, &cfg); + nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0); + lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns; + check("suspend on: a program does not preempt", lat, (2000000 - 1000) + 40000); + nand_media_destroy(&m); + + /* plane gate without a channel (ZNS default): a read preempts an erase */ + zns_config(&cfg); + cfg.policy.channel_mode = NAND_CH_OFF; + cfg.policy.pe_suspend = true; + cfg.timing.tsusp_ns = 5000; + nand_media_init(&m, &cfg); + nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0); + lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns; + check("plane gate: read preempts the erase", lat, 5000 + 65000); + lat = nand_media_op(&m, &a, NAND_MEDIA_PROGRAM, t0 + 1000).latency_ns; + check("plane gate: erase resumes after the read", + lat, (2000000 + 65000 + 5000 - 1000) + 450000); + nand_media_destroy(&m); + + /* staged channel: the existing suspend branch, now reachable */ + bb_config(&cfg); + cfg.policy.channel_mode = NAND_CH_STAGED; + cfg.timing.cmd_addr_ns = 300; + cfg.timing.page_xfer_ns = 20000; + cfg.timing.status_ns = 100; + cfg.policy.pe_suspend = true; + cfg.timing.tsusp_ns = 5000; + nand_media_init(&m, &cfg); + nand_media_op(&m, &a, NAND_MEDIA_ERASE, t0); + lat = nand_media_op(&m, &a, NAND_MEDIA_READ, t0 + 1000).latency_ns; + /* command 300, then suspend 5000 + array 10000, then data-out 20000 + * (the bus is idle: the erase moved no data) and status 100 */ + check("staged: read = cmd + tsusp + array + xfer + status", + lat, 300 + 5000 + 10000 + 20000 + 100); + nand_media_destroy(&m); +} + int main(void) { /* @@ -441,6 +533,7 @@ int main(void) test_plane_gate_with_channel(); test_multiplane_erase(); test_copyback(); + test_pe_suspend(); printf("1..%d\n", ntests); return failures ? 1 : 0; } diff --git a/hw/femu/zns/zftl.c b/hw/femu/zns/zftl.c index 985bc791c41..2061fc20520 100644 --- a/hw/femu/zns/zftl.c +++ b/hw/femu/zns/zftl.c @@ -118,6 +118,8 @@ void zns_nand_media_init(struct zns_ssd *zns) cfg.timing.cmd_addr_ns = zns->timing.cmd_addr_lat; cfg.timing.page_xfer_ns = zns->timing.pg_xfer_lat; cfg.timing.status_ns = zns->timing.status_lat; + cfg.policy.pe_suspend = zns->timing.pe_suspend; + cfg.timing.tsusp_ns = zns->timing.tsusp_ns; cfg.policy.channel_mode = (cfg.timing.cmd_addr_ns || cfg.timing.page_xfer_ns || cfg.timing.status_ns) ? NAND_CH_STAGED : NAND_CH_OFF; cfg.timeline = &zns_timeline_ops; diff --git a/hw/femu/zns/zns.c b/hw/femu/zns/zns.c index b4d0c1644ae..a61096393b9 100644 --- a/hw/femu/zns/zns.c +++ b/hw/femu/zns/zns.c @@ -1857,6 +1857,9 @@ static void zns_init_params(FemuCtrl *n, NvmeNamespace *ns) id_zns->timing.cmd_addr_lat = n->zns_params.zns_cmd_addr_lat; id_zns->timing.pg_xfer_lat = n->zns_params.zns_pg_xfer_lat; id_zns->timing.status_lat = n->zns_params.zns_status_lat; + /* P/E suspend: default off, so the plane gate is bit-identical to before */ + id_zns->timing.pe_suspend = (n->zns_params.zns_pe_suspend != 0); + id_zns->timing.tsusp_ns = n->zns_params.zns_tsusp_ns; /* * Optional write-fault injection. One write in N fails and takes its zone @@ -1976,7 +1979,7 @@ static bool zns_check_params(FemuCtrl *n, NvmeNamespace *ns, Error **errp) } if (p->zns_pg_rd_lat < 0 || p->zns_pg_wr_lat < 0 || p->zns_blk_er_lat < 0 || p->zns_cmd_addr_lat < 0 || p->zns_pg_xfer_lat < 0 || - p->zns_status_lat < 0) { + p->zns_status_lat < 0 || p->zns_tsusp_ns < 0) { error_setg(errp, "zns NAND timing knobs must not be negative"); return false; } diff --git a/hw/femu/zns/zns.h b/hw/femu/zns/zns.h index 4987a5cfd33..cebc3e1d13d 100644 --- a/hw/femu/zns/zns.h +++ b/hw/femu/zns/zns.h @@ -124,6 +124,9 @@ typedef struct SSDNandFlashTiming { uint64_t cmd_addr_lat; /* command and address cycles */ uint64_t pg_xfer_lat; /* page data-in (program) / data-out (read) */ uint64_t status_lat; /* status read after a read or erase */ + /* program/erase suspend: a read preempts a busy plane (0 = off) */ + bool pe_suspend; + uint64_t tsusp_ns; } SSDNandFlashTiming; struct zns_write_cache{