From ed818502156c3afccf6304ecc5d2a642d7ad1f69 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 20 Oct 2026 12:23:47 +0300 Subject: [PATCH 1/3] feat: retry a selection past 500 in rounds instead of refusing it --- .../Resources/Xchanges/BulkRetry.cs | 4 +- .../Resources/Xchanges/BulkRetryPlanner.cs | 29 ++++-- .../Resources/Xchanges/BulkRetryPreview.cs | 7 +- .../Tests/RetryChainTests.cs | 68 ++++++++++++++ SW.Bitween.Sdk/Model/Xchange.cs | 21 ++++- .../ClientApp/src/api/http/exchanges.ts | 4 + SW.Bitween.Web/ClientApp/src/api/types.ts | 7 +- .../src/pages/exchanges/ExchangesPage.tsx | 58 +++++++++++- .../__tests__/ExchangesList.test.tsx | 88 ++++++++++++++++++- .../ClientApp/src/pages/exchanges/shared.tsx | 72 ++++++++++++--- docs/api-reference.md | 2 +- docs/retries-and-alerts.md | 2 +- 12 files changed, 328 insertions(+), 34 deletions(-) diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs index 547a9e62..65913c5f 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetry.cs @@ -32,13 +32,13 @@ public async Task Handle(XchangeBulkRetry request) { await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Exchanges.Operate); - var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request); + var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request, BulkRetryPlanner.Limit); var plan = prepared.Plan; if (plan.OverLimit) throw new SWValidationException("TOO_MANY", $"{plan.Selected:n0} exchanges is more than the {BulkRetryPlanner.Limit} this can retry " + - "in one go. Narrow the filter and retry the rest after."); + $"in one go. Send the preview's targets back in rounds of {BulkRetryPlanner.Limit}."); var retried = 0; foreach (var id in prepared.Targets) diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs index ec1064d4..0a4d0150 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs @@ -23,10 +23,21 @@ internal sealed class BulkRetryPlanner /// The largest selection one request will carry out. Every exchange retried means reading its /// input back out of storage and writing a new exchange, in sequence, inside the one request - /// so the ceiling is about what can finish before something upstream gives up waiting, not - /// about the database. Past it the caller is asked to narrow the filter. + /// about the database. A bigger selection is retried in rounds of this many. /// internal const int Limit = 500; + /// + /// The largest selection a preview works out. Up to it, the plan lists every attempt that will + /// run, and the caller sends them back in rounds of . Past it the caller is + /// asked to narrow the filter. + /// + /// + /// Rounds rather than raising : each round is its own request, so none of them + /// runs long, and a round that fails leaves the ones before it done. + /// + internal const int MaxSelection = 5000; + /// /// Guards the walk to the newest attempt against a cycle in the data, the same way /// does. No honest chain approaches it. @@ -44,24 +55,29 @@ internal sealed class Prepared internal List Targets { get; init; } = new List(); } - internal async Task Prepare(XchangeBulkRetry request) + /// + /// The largest selection to work out: for a retry, + /// for a preview. + /// + internal async Task Prepare(XchangeBulkRetry request, int upTo) { var selection = new XchangeSelection(_dbContext); - var selected = await selection.Resolve(request, Limit); + var selected = await selection.Resolve(request, upTo); - if (selected.Count > Limit) + if (selected.Count > upTo) return new Prepared { Plan = new XchangeBulkRetryPlan { // A hand-picked list is already counted. A filter's selection was cut off at - // Limit + 1 to notice it was too long without reading it all, so that one is + // upTo + 1 to notice it was too long without reading it all, so that one is // counted properly now - the caller is told how far past the line it is rather // than just "501", and never a count of rows it did not ask about. Selected = string.IsNullOrWhiteSpace(request.Filter) ? selected.Count : await selection.Count(request), Limit = Limit, + MaxSelection = MaxSelection, OverLimit = true } }; @@ -70,7 +86,7 @@ internal async Task Prepare(XchangeBulkRetry request) var state = await ReadState(newestAttempt.Values.Distinct().ToList()); var reset = request.Reset; - var plan = new XchangeBulkRetryPlan { Selected = selected.Count, Limit = Limit }; + var plan = new XchangeBulkRetryPlan { Selected = selected.Count, Limit = Limit, MaxSelection = MaxSelection }; var targets = new List(); foreach (var selectedId in selected) @@ -98,6 +114,7 @@ internal async Task Prepare(XchangeBulkRetry request) } plan.WillRetry = targets.Count; + plan.Targets = targets; plan.Properties = await ReadPromotedProperties(plan); return new Prepared { Plan = plan, Targets = targets }; } diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs index dc77a50f..b9cc69bf 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPreview.cs @@ -12,6 +12,11 @@ namespace SW.Bitween.Resources.Xchanges; /// ones that have since succeeded or are still running drop out, and two selections in one chain /// come to the same attempt. Retrying is not undoable, so the caller gets to show all of that and /// be told the number before anyone commits to it. +/// +/// It works out a selection past what one retry carries out, up to +/// , so the caller can retry it in rounds from the plan's +/// targets. +/// /// [HandlerName("bulkretrypreview")] public class BulkRetryPreview : ICommandHandler @@ -31,7 +36,7 @@ public async Task Handle(XchangeBulkRetry request) // describes an action only they can take, so it is gated with the action. await _requestContext.EnsurePermission(_dbContext, Model.Permissions.Exchanges.Operate); - var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request); + var prepared = await new BulkRetryPlanner(_dbContext).Prepare(request, BulkRetryPlanner.MaxSelection); return prepared.Plan; } } diff --git a/SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs b/SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs index 56d07490..ef64f8f0 100644 --- a/SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs +++ b/SW.Bitween.IntegrationTests/Tests/RetryChainTests.cs @@ -317,6 +317,74 @@ public async Task BulkRetryPreview_describes_the_same_work_without_doing_it() Assert.False(await db.Set().AnyAsync(x => x.RetryFor == second.Id)); } + /// + /// A selection bigger than one retry is sent back in rounds of the preview's targets: the attempts + /// themselves, already moved to the end of their chains, so a round retries exactly those. + /// + [Fact] + public async Task The_preview_lists_the_attempts_a_round_retries() + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var xs = scope.ServiceProvider.GetRequiredService(); + var ctx = scope.Superuser(); + + var (sub, retried) = await FailedXchange(db, xs, "Targets Doc"); + var newest = await RetryOnce(db, xs, ctx, retried.Id); + var other = await xs.CreateXchange(sub, new XchangeFile("{}")); + await db.SaveChangesAsync(); + db.Set().Add(new XchangeResult(other.Id, null, null, exception: "boom")); + await db.SaveChangesAsync(); + + var plan = (XchangeBulkRetryPlan)await new Resources.Xchanges.BulkRetryPreview(db, ctx) + .Handle(new XchangeBulkRetry { Ids = [retried.Id, newest.Id, other.Id] }); + + Assert.Equal(new[] { newest.Id, other.Id }, plan.Targets); + + var round = (XchangeBulkRetryPlan)await new Resources.Xchanges.BulkRetry(db, ctx, xs) + .Handle(new XchangeBulkRetry { Ids = plan.Targets }); + await db.SaveChangesAsync(); + + Assert.Equal(2, round.WillRetry); + Assert.Empty(round.Substituted); + Assert.True(await db.Set().AnyAsync(x => x.RetryFor == newest.Id)); + Assert.True(await db.Set().AnyAsync(x => x.RetryFor == other.Id)); + } + + /// + /// Past what one retry carries out, the preview still works the selection out so it can be + /// retried in rounds, while the retry itself takes no more than one round. Ids that don't exist + /// are enough to count with: the size is checked before any exchange is read. + /// + [Fact] + public async Task A_selection_past_one_retry_is_planned_whole_but_retried_a_round_at_a_time() + { + await using var scope = _fixture.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var xs = scope.ServiceProvider.GetRequiredService(); + var ctx = scope.Superuser(); + var preview = new Resources.Xchanges.BulkRetryPreview(db, ctx); + var limits = (XchangeBulkRetryPlan)await preview.Handle(new XchangeBulkRetry { Ids = [] }); + Assert.True(limits.MaxSelection > limits.Limit); + + static System.Collections.Generic.List Ids(int count) => + Enumerable.Range(0, count).Select(_ => Guid.NewGuid().ToString("N")).ToList(); + + var pastOneRetry = Ids(limits.Limit + 1); + var plan = (XchangeBulkRetryPlan)await preview.Handle(new XchangeBulkRetry { Ids = pastOneRetry }); + Assert.False(plan.OverLimit); + Assert.Equal(limits.Limit + 1, plan.Skipped.Count); + + var refused = await Assert.ThrowsAsync(() => + new Resources.Xchanges.BulkRetry(db, ctx, xs).Handle(new XchangeBulkRetry { Ids = pastOneRetry })); + Assert.Contains(refused.Validations, v => v.Key == "TOO_MANY"); + + var pastAnyRetry = (XchangeBulkRetryPlan)await preview.Handle( + new XchangeBulkRetry { Ids = Ids(limits.MaxSelection + 1) }); + Assert.True(pastAnyRetry.OverLimit); + Assert.Equal(limits.MaxSelection + 1, pastAnyRetry.Selected); + } + /// /// Retrying is an operator's action, and until this was added the endpoints took anyone's word /// for it: the UI hid the button without a permission, but the API accepted the call from any diff --git a/SW.Bitween.Sdk/Model/Xchange.cs b/SW.Bitween.Sdk/Model/Xchange.cs index a15c2732..902ba788 100644 --- a/SW.Bitween.Sdk/Model/Xchange.cs +++ b/SW.Bitween.Sdk/Model/Xchange.cs @@ -113,16 +113,29 @@ public class XchangeBulkRetryPlan /// Distinct exchanges that will be (or were) retried. public int WillRetry { get; set; } - /// The largest selection this endpoint will carry out in one request. + /// + /// The largest selection one retry request carries out. A bigger one is retried in rounds of + /// this many, from . + /// public int Limit { get; set; } + /// The largest selection a preview works out; past it, narrow the selection. + public int MaxSelection { get; set; } + /// - /// true when the selection is past . Nothing else is filled in - /// then - resolving thousands of chains to describe a request that will be refused - /// costs more than the answer is worth. + /// true when the selection is past for a preview, or past + /// for a retry. Nothing else is filled in then - resolving thousands of + /// chains to describe a request that will be refused costs more than the answer is worth. /// public bool OverLimit { get; set; } + /// + /// The attempts that will run: each selection's newest attempt, once per chain, minus the + /// skipped. Sent back as ids in rounds of when there are + /// more than one retry carries out. + /// + public List Targets { get; set; } = new List(); + /// /// Selections that had already been retried, and the later attempt standing in for each. /// diff --git a/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts b/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts index 663805ca..5b06859e 100644 --- a/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts +++ b/SW.Bitween.Web/ClientApp/src/api/http/exchanges.ts @@ -79,7 +79,9 @@ interface RawBulkRetryPlan { selected: number; willRetry: number; limit: number; + maxSelection: number; overLimit: boolean; + targets: string[] | null; substituted: { selectedId: string; retryId: string }[] | null; skipped: { id: string; reason: string }[] | null; properties: Record> | null; @@ -262,7 +264,9 @@ async function postBulkRetry( selected: plan.selected, willRetry: plan.willRetry, limit: plan.limit, + maxSelection: plan.maxSelection, overLimit: plan.overLimit, + targets: plan.targets ?? [], substituted: plan.substituted ?? [], skipped: plan.skipped ?? [], properties: plan.properties ?? {}, diff --git a/SW.Bitween.Web/ClientApp/src/api/types.ts b/SW.Bitween.Web/ClientApp/src/api/types.ts index 2f5f2e1c..881839b0 100644 --- a/SW.Bitween.Web/ClientApp/src/api/types.ts +++ b/SW.Bitween.Web/ClientApp/src/api/types.ts @@ -1544,9 +1544,14 @@ export interface ExportFilesCheck { export interface BulkRetryPlan { selected: number; willRetry: number; + /** The most one retry request carries out; a bigger selection goes in rounds of this many. */ limit: number; - /** True when the selection is past `limit`; nothing else is filled in. */ + /** The most a preview works out. */ + maxSelection: number; + /** True when the selection is past `maxSelection` (or, for a retry, past `limit`); nothing else is filled in. */ overLimit: boolean; + /** The attempts that will run - what the rounds send back as ids when `selected` is past `limit`. */ + targets: string[]; substituted: RetrySubstitution[]; skipped: RetrySkip[]; /** diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx index 77204025..91db4a5b 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx @@ -1,9 +1,10 @@ -import { useCallback, useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { Link, useSearchParams } from "react-router"; import { keepPreviousData, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { ArrowUp, Download, Pause, Play, Plus, RefreshCw, RotateCcw, TriangleAlert, X } from "lucide-react"; import { api, + type BulkRetryPlan, type BulkRetrySelection, type ExchangeStatus, type ExportFilesCheck, @@ -304,20 +305,66 @@ export function ExchangesPage() { staleTime: 30_000, }); + /** How far a retry in rounds has got. */ + const [bulkProgress, setBulkProgress] = useState<{ done: number; total: number } | null>(null); + /** Set when the confirm is closed mid-way: the rounds stop after the one under way. */ + const stopRounds = useRef(false); + + /** + * A selection past what one request carries out: the preview listed every attempt that will run, + * and they go back as ids, a round at a time. No request runs long, and a round that fails leaves + * the ones before it done. The attempts, not the selection, so a round can't reach past them to an + * attempt an earlier round just made. + */ + async function retryInRounds(planned: BulkRetryPlan, reset: boolean) { + const done = { ...planned, willRetry: 0, skipped: [...planned.skipped], stoppedAt: null as number | null }; + const total = planned.targets.length; + stopRounds.current = false; + setBulkProgress({ done: 0, total }); + for (let at = 0; at < total; at += planned.limit) { + if (stopRounds.current) { + done.stoppedAt = at; + break; + } + let round: BulkRetryPlan; + try { + round = await api.bulkRetryExchanges({ ids: planned.targets.slice(at, at + planned.limit) }, { reset }); + } catch (e) { + throw new Error( + `${done.willRetry.toLocaleString()} of ${total.toLocaleString()} were retried, then a round failed` + + ` (${e instanceof Error ? e.message : String(e)}). The rest were not retried.`, + ); + } + done.willRetry += round.willRetry; + done.skipped.push(...round.skipped); + setBulkProgress({ done: Math.min(at + planned.limit, total), total }); + } + return done; + } + const bulkRetry = useMutation({ - mutationFn: (reset: boolean) => api.bulkRetryExchanges(selection, { reset }), + mutationFn: (reset: boolean) => + plan && plan.selected > plan.limit + ? retryInRounds(plan, reset) + : api.bulkRetryExchanges(selection, { reset }).then((done) => ({ ...done, stoppedAt: null })), onSuccess: (done) => { setBulkConfirm(false); setBulkReset(false); + setBulkProgress(null); clearSelection(); setBulkResult( `${done.willRetry.toLocaleString()} retr${done.willRetry === 1 ? "y" : "ies"} started` + (done.substituted.length > 0 ? `, ${done.substituted.length} continuing an existing chain` : "") + (done.skipped.length > 0 ? `, ${done.skipped.length} skipped` : "") + + (done.stoppedAt != null + ? `. Stopped after ${done.stoppedAt.toLocaleString()} of ${done.targets.length.toLocaleString()}; the rest were not retried` + : "") + ".", ); void queryClient.invalidateQueries({ queryKey: keys.exchanges.all }); }, + // The list changed as far as the rounds got. + onError: () => void queryClient.invalidateQueries({ queryKey: keys.exchanges.all }), }); // Asked first, so someone hears that some exchanges are past the files storage keeps before choosing to @@ -654,10 +701,17 @@ export function ExchangesPage() { plan={plan ?? null} planLoading={planLoading} busy={bulkRetry.isPending} + progress={bulkProgress} + error={bulkRetry.error?.message ?? null} onConfirm={(reset) => bulkRetry.mutate(reset)} onResetChange={setBulkReset} onClose={() => { + stopRounds.current = true; setBulkConfirm(false); + setBulkProgress(null); + // Not while rounds are still under way: Retry would be offered again, and a second run + // beside the first would retry what the first just did. + if (bulkRetry.isError) bulkRetry.reset(); // The dialog starts unticked each time it opens, so the mirrored copy has to as well. setBulkReset(false); }} diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx index b042c64d..0bf64158 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx @@ -140,13 +140,13 @@ describe("the exchanges list", () => { it.each([ { - answer: "past the cap, it refuses", - plan: { selected: 60, willRetry: 0, limit: 50, overLimit: true }, - says: /more than the 50 a single retry will carry out/, + answer: "past what rounds can take, it refuses", + plan: { selected: 6000, willRetry: 0, limit: 500, maxSelection: 5000, overLimit: true }, + says: /more than the 5,000 one retry can work through/, }, { answer: "with nothing retryable, it says so", - plan: { selected: 60, willRetry: 0, limit: 500, overLimit: false }, + plan: { selected: 60, willRetry: 0, limit: 500, maxSelection: 5000, overLimit: false }, says: /^Nothing here can be retried\.$/, }, ])("$answer, and offers only Close", async ({ plan, says }) => { @@ -170,6 +170,86 @@ describe("the exchanges list", () => { await user.click(within(dialog).getByText("Close", { selector: "button" })); expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); }); + + /** + * Past what one retry carries out, the confirm doesn't send the selection: it sends the attempts + * the preview listed, a round at a time. Here the limit is 2, so 5 attempts are 3 rounds. + */ + describe("past one retry, in rounds", () => { + const targets = [11, 12, 13, 14, 15].map((n) => exchange(n).id); + const preview = http.post(apiPath("/xchanges/bulkretrypreview"), () => + HttpResponse.json({ + selected: 59, + willRetry: 5, + limit: 2, + maxSelection: 5000, + overLimit: false, + targets, + substituted: [], + skipped: [{ id: exchange(16).id, reason: "It succeeded, so there is nothing to retry." }], + properties: {}, + }), + ); + /** + * The retry endpoint, keeping every round it was sent; `fails` answers that round with a 500, + * and `holds` keeps that round waiting until `release` is called. + */ + function rounds({ fails, holds }: { fails?: number; holds?: number } = {}) { + const sent: { ids: string[] }[] = []; + let release = () => {}; + const held = new Promise((resolve) => (release = resolve)); + const handler = http.post(apiPath("/xchanges/bulkretry"), async ({ request }) => { + const body = (await request.json()) as { ids: string[] }; + sent.push(body); + if (sent.length === holds) await held; + if (sent.length === fails) return HttpResponse.json({ message: "Broker unreachable" }, { status: 500 }); + const n = body.ids.length; + return HttpResponse.json({ selected: n, willRetry: n, limit: 2, maxSelection: 5000, overLimit: false, targets: body.ids }); + }); + return { handler, sent, release: () => release() }; + } + + it("sends the planned attempts a round at a time, then says what ran", async () => { + const retry = rounds({ holds: 2 }); + const { user } = renderApp("/exchanges?status=failed", { + handlers: [exchanges({ total: 60 }).handler, ...filterOptions, preview, retry.handler], + }); + + await selectAllMatching(user); + await user.click(screen.getByRole("button", { name: "Retry selected…" })); + const dialog = await screen.findByRole("dialog"); + expect(await within(dialog).findByText(/they go in 3 rounds of up to 2/)).toBeVisible(); + + await user.click(within(dialog).getByRole("button", { name: "Retry" })); + // While the second round is out, the first one's attempts are counted. + expect(await within(dialog).findByText("Retried 2 of 5…")).toBeVisible(); + expect(within(dialog).getByRole("progressbar", { name: "Retried so far" })).toHaveAttribute("aria-valuenow", "2"); + retry.release(); + + expect(await screen.findByText("5 retries started, 1 skipped.")).toBeVisible(); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + expect(retry.sent.map((r) => r.ids)).toEqual([targets.slice(0, 2), targets.slice(2, 4), targets.slice(4)]); + }); + + it("stops at a round that fails, says how far it got, and doesn't offer to go again", async () => { + const retry = rounds({ fails: 2 }); + const { user } = renderApp("/exchanges?status=failed", { + handlers: [exchanges({ total: 60 }).handler, ...filterOptions, preview, retry.handler], + }); + + await selectAllMatching(user); + await user.click(screen.getByRole("button", { name: "Retry selected…" })); + const dialog = await screen.findByRole("dialog"); + await within(dialog).findByText(/they go in 3 rounds/); + await user.click(within(dialog).getByRole("button", { name: "Retry" })); + + expect(await within(dialog).findByRole("alert")).toHaveTextContent( + /^2 of 5 were retried, then a round failed .*The rest were not retried\.$/, + ); + expect(retry.sent).toHaveLength(2); + expect(within(dialog).queryByRole("button", { name: "Retry" })).not.toBeInTheDocument(); + }); + }); }); /** diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/shared.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/shared.tsx index 387907c7..39982b85 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/shared.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/shared.tsx @@ -137,12 +137,17 @@ function ExchangeIdentity({ * selection is rarely just itself: exchanges already retried hand over to their newest attempt, * ones that have since succeeded drop out, and two selections in one chain come to the same * attempt. All of that is shown before anyone commits, since a retry cannot be taken back. + * + * A selection past what one retry request carries out goes in rounds; `progress` says how far + * they have got. */ export function RetryDialog({ count, plan, planLoading = false, busy, + progress = null, + error = null, onConfirm, onResetChange, onClose, @@ -152,6 +157,10 @@ export function RetryDialog({ plan?: BulkRetryPlan | null; planLoading?: boolean; busy: boolean; + /** Bulk retries in rounds: the attempts retried so far, of all that will run. */ + progress?: { done: number; total: number } | null; + /** Why the retry stopped, when it did. */ + error?: string | null; onConfirm: (reset: boolean) => void; /** * Bulk retries only: the plan depends on this choice - re-resolving properties is impossible @@ -163,6 +172,7 @@ export function RetryDialog({ const [reset, setReset] = useState(false); const bulk = count !== 1; const nothingToDo = plan != null && !plan.overLimit && plan.willRetry === 0; + const rounds = plan != null && plan.selected > plan.limit ? Math.ceil(plan.willRetry / plan.limit) : 0; return ( {plan.selected.toLocaleString()} exchanges match this filter, which is more than the{" "} - {plan.limit.toLocaleString()} a single retry will carry out. Narrow the filter - by + {plan.maxSelection.toLocaleString()} one retry can work through. Narrow the filter - by status, partner or date - and retry the rest after.

) : ( @@ -192,6 +202,34 @@ export function RetryDialog({ }.`}

+ {rounds > 1 && !progress && ( +

+ That's more than one retry carries out, so they go in {rounds} rounds of up to{" "} + {plan!.limit.toLocaleString()}, one after another. Keep this page open until they finish. +

+ )} + + {progress && ( +
+

+ Retried {progress.done.toLocaleString()} of {progress.total.toLocaleString()}… +

+
+
+
+
+ )} + {plan != null && plan.substituted.length > 0 && (

@@ -240,23 +278,33 @@ export function RetryDialog({

)} - { - setReset(e.target.checked); - onResetChange?.(e.target.checked); - }} - /> + {/* Not once rounds are under way: they keep the choice they started with. */} + {!progress && ( + { + setReset(e.target.checked); + onResetChange?.(e.target.checked); + }} + /> + )} )} + {error && ( +

+ {error} +

+ )} +
- {!plan?.overLimit && !nothingToDo && ( + {/* Not again after a failure: the plan it would repeat is out of date by the rounds already done. */} + {!plan?.overLimit && !nothingToDo && !error && ( diff --git a/docs/api-reference.md b/docs/api-reference.md index e603b7b8..43a88a50 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -100,7 +100,7 @@ Exchange search filters, besides ordinary fields: | `StatusFilter:1:{n}` | 0 processing, 1 success, 2 bad response, 3 failed. Other values are refused. | | `LatestOnly:1:true` | Only the newest attempt of each retry chain | -Bulk retry takes `{ ids, filter, excludeIds, reason, reset }`. `filter` is the same query-string fragment the search takes, such as `filter=StatusFilter:1:3&filter=LatestOnly:1:true`, and it replaces `ids` when present. The plan returned by both endpoints lists how many were selected and will be retried, attempts substituted with the end of their chain, and skipped exchanges with reasons. At most 500 exchanges can be retried at once. +Bulk retry takes `{ ids, filter, excludeIds, reason, reset }`. `filter` is the same query-string fragment the search takes, such as `filter=StatusFilter:1:3&filter=LatestOnly:1:true`, and it replaces `ids` when present. The plan returned by both endpoints lists how many were selected and will be retried, attempts substituted with the end of their chain, and skipped exchanges with reasons, plus `targets`, the attempts that will run. The retry carries out at most `limit` (500) per request; the preview plans up to `maxSelection` (5,000), so a larger selection is retried by sending `targets` back as `ids`, `limit` at a time. Export takes `{ ids, filter, excludeIds }`, chosen the same way, and streams back a zip (`application/zip`) with a folder per exchange and a `missing.txt` listing files that storage no longer has. At most 500 exchanges go in one zip; a larger selection is refused with `TOO_MANY`. `export/check` takes the same body and returns `{ count, withoutFiles, keptDays }`: how many exchanges the zip would take, how many of them are older than the bucket's rule keeps files, and that rule's days when one covers them all. diff --git a/docs/retries-and-alerts.md b/docs/retries-and-alerts.md index 327be913..19be7445 100644 --- a/docs/retries-and-alerts.md +++ b/docs/retries-and-alerts.md @@ -22,7 +22,7 @@ Content-Type: application/json `POST /api/xchanges/bulkretry` retries either a list of `ids`, or everything a search `filter` matches minus `excludeIds`. `POST /api/xchanges/bulkretrypreview` returns the same plan without running it, and the UI shows that plan before you confirm. -- A bulk retry covers at most 500 exchanges. A larger selection is refused, so narrow the filter. +- One retry request carries out at most 500 exchanges. The preview works out up to 5,000 and lists the attempts that will run in `targets`; the UI sends them back as `ids` in rounds of 500 and shows how far it has got. If a round fails, the rounds before it are done and the rest are not retried. Past 5,000, narrow the filter. - An exchange that was already retried is replaced by the newest attempt of its chain. Two selections that lead to the same attempt are retried once. - The plan skips, with a reason, exchanges that no longer exist, that succeeded, that have an automatic retry waiting, whose subscription is gone when `reset` is on, or whose input file cannot be read. - Exchanges with no result yet are included, so work stranded by an outage can be recovered. Make sure nothing is still processing before retrying it. From 64f4dc5ec68f777a4f0f492cf0c43138b0967a53 Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 20 Oct 2026 12:39:56 +0300 Subject: [PATCH 2/3] perf: track bulk retry targets in a set --- SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs index 0a4d0150..8fc46cf0 100644 --- a/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs +++ b/SW.Bitween.Api/Resources/Xchanges/BulkRetryPlanner.cs @@ -88,6 +88,8 @@ internal async Task Prepare(XchangeBulkRetry request, int upTo) var plan = new XchangeBulkRetryPlan { Selected = selected.Count, Limit = Limit, MaxSelection = MaxSelection }; var targets = new List(); + // The list keeps the order; the set answers "already in it" without walking up to MaxSelection entries. + var targeted = new HashSet(); foreach (var selectedId in selected) { @@ -109,7 +111,7 @@ internal async Task Prepare(XchangeBulkRetry request, int upTo) // Two selections in the same chain - an exchange and its own retry, say - come to the // same attempt, which is retried once. - if (!targets.Contains(targetId)) + if (targeted.Add(targetId)) targets.Add(targetId); } From e900de8e180d40bea0371b0c5c136644a75b0cff Mon Sep 17 00:00:00 2001 From: Hamza Alqurneh Date: Tue, 20 Oct 2026 12:39:56 +0300 Subject: [PATCH 3/3] fix: a bulk retry finishing after its dialog closed leaves a newer one alone --- .../src/pages/exchanges/ExchangesPage.tsx | 17 ++++++++++--- .../__tests__/ExchangesList.test.tsx | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx index 91db4a5b..504dbfc0 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/ExchangesPage.tsx @@ -309,6 +309,11 @@ export function ExchangesPage() { const [bulkProgress, setBulkProgress] = useState<{ done: number; total: number } | null>(null); /** Set when the confirm is closed mid-way: the rounds stop after the one under way. */ const stopRounds = useRef(false); + /** + * The run the open confirm belongs to. Closing it moves this on, so a run that ends afterwards + * reports what it did but can't close a confirm opened since, or clear a selection made since. + */ + const bulkRun = useRef(0); /** * A selection past what one request carries out: the preview listed every attempt that will run, @@ -347,11 +352,14 @@ export function ExchangesPage() { plan && plan.selected > plan.limit ? retryInRounds(plan, reset) : api.bulkRetryExchanges(selection, { reset }).then((done) => ({ ...done, stoppedAt: null })), - onSuccess: (done) => { - setBulkConfirm(false); - setBulkReset(false); + onMutate: () => ({ run: ++bulkRun.current }), + onSuccess: (done, _reset, started) => { setBulkProgress(null); - clearSelection(); + if (started?.run === bulkRun.current) { + setBulkConfirm(false); + setBulkReset(false); + clearSelection(); + } setBulkResult( `${done.willRetry.toLocaleString()} retr${done.willRetry === 1 ? "y" : "ies"} started` + (done.substituted.length > 0 ? `, ${done.substituted.length} continuing an existing chain` : "") + @@ -707,6 +715,7 @@ export function ExchangesPage() { onResetChange={setBulkReset} onClose={() => { stopRounds.current = true; + bulkRun.current++; setBulkConfirm(false); setBulkProgress(null); // Not while rounds are still under way: Retry would be offered again, and a second run diff --git a/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx index 0bf64158..d45757f9 100644 --- a/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx +++ b/SW.Bitween.Web/ClientApp/src/pages/exchanges/__tests__/ExchangesList.test.tsx @@ -231,6 +231,31 @@ describe("the exchanges list", () => { expect(retry.sent.map((r) => r.ids)).toEqual([targets.slice(0, 2), targets.slice(2, 4), targets.slice(4)]); }); + it("closed mid-way, stops after the round under way and leaves a confirm opened since alone", async () => { + const retry = rounds({ holds: 1 }); + const { user } = renderApp("/exchanges?status=failed", { + handlers: [exchanges({ total: 60 }).handler, ...filterOptions, preview, retry.handler], + }); + + await selectAllMatching(user); + await user.click(screen.getByRole("button", { name: "Retry selected…" })); + const dialog = await screen.findByRole("dialog"); + await within(dialog).findByText(/they go in 3 rounds/); + await user.click(within(dialog).getByRole("button", { name: "Retry" })); + await user.click(within(dialog).getByRole("button", { name: "Cancel" })); + + await user.click(screen.getByRole("button", { name: "Retry selected…" })); + await screen.findByRole("dialog"); + retry.release(); + + expect( + await screen.findByText("2 retries started, 1 skipped. Stopped after 2 of 5; the rest were not retried."), + ).toBeVisible(); + expect(retry.sent).toHaveLength(1); + expect(screen.getByRole("dialog")).toBeVisible(); + expect(screen.getByRole("button", { name: "Retry selected…" })).toBeVisible(); + }); + it("stops at a round that fails, says how far it got, and doesn't offer to go again", async () => { const retry = rounds({ fails: 2 }); const { user } = renderApp("/exchanges?status=failed", {