diff --git a/CHANGES.md b/CHANGES.md index 99da4b268..fd4eb16b7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -176,6 +176,11 @@ To be released. ### @fedify/backfill + - Added `backfillActivities()` to stream original conversation activities, + including updates and deletions, in collection order without requiring + their targets to resolve. It reuses collection source selection and + traversal controls while keeping existing `backfill()` post output + unchanged. [[#1251], [#1447] by Jiwon Kwon\] - Added `contextDiscovery: "ancestors"` to backfill options to find the nearest usable conversation collection through `inReplyTo` ancestors when the seed has none. Discovery respects `maxDepth`, shares traversal budgets @@ -193,8 +198,10 @@ To be released. HTTPS identifiers. Errors cloning embedded history propagate instead of triggering fallback. [[#1250], [#1396] by Jiwon Kwon\] +[#1251]: https://github.com/fedify-dev/fedify/issues/1251 [#1252]: https://github.com/fedify-dev/fedify/issues/1252 [#1416]: https://github.com/fedify-dev/fedify/pull/1416 +[#1447]: https://github.com/fedify-dev/fedify/pull/1447 ### @fedify/cfworkers diff --git a/changes.d/backfill/activity-stream.md b/changes.d/backfill/activity-stream.md new file mode 100644 index 000000000..c8694098f --- /dev/null +++ b/changes.d/backfill/activity-stream.md @@ -0,0 +1,10 @@ +--- +links: + '#1251': https://github.com/fedify-dev/fedify/issues/1251 + '#1447': https://github.com/fedify-dev/fedify/pull/1447 +--- + - Added `backfillActivities()` to stream original conversation activities, + including updates and deletions, in collection order without requiring + their targets to resolve. It reuses collection source selection and + traversal controls while keeping existing `backfill()` post output + unchanged. [[#1251], [#1447] by Jiwon Kwon] diff --git a/claude-plugin/skills/fedify/SKILL.md b/claude-plugin/skills/fedify/SKILL.md index 548e932f3..b9dc19690 100644 --- a/claude-plugin/skills/fedify/SKILL.md +++ b/claude-plugin/skills/fedify/SKILL.md @@ -344,21 +344,40 @@ history collection is resolved, it stays selected even if empty or no items can be yielded. Later history references are not tried, and sources are not merged. Portable history references are passed to the configured backfill loader for resolution and verification. -Existing strategies control post extraction; later explicitly configured -strategies can still run. Set `contextDiscovery: "ancestors"` to search -`inReplyTo` ancestors when the seed has no usable collection, bounded by + +Use `backfillActivities(context, seed, options)` for original conversation +activities such as `Create`, `Update`, `Delete`, and `Like`. It reuses +`BackfillContext` and returns `BackfillItem` independently of the +seed's type. `BackfillActivityOptions` pick `contextSource`, `contextDiscovery`, +`maxDepth`, `maxItems`, `maxRequests`, `interval`, and `signal` from existing +options; they expose neither `strategies` nor `output`. Ancestor discovery +only locates a collection, and reply-tree output is unavailable. Activities +are yielded in collection order without fetching their targets; direct posts +are skipped and wrappers such as `Add` remain intact. Deduplicate by activity +ID, count yielded activities toward `maxItems`, and use metadata +`strategy: "context-activities"`, `origin: "collection"`, and `depth: 0`. +Applications must authorize activities before applying them: discovery is not +verified inbox delivery. Choose this function directly without first calling +the post API; each invocation has its own traversal cache and budget. +Vocabulary collections can separately cache items and skipped-load +placeholders. A loader reusing the same collection instance can retain those +across calls; return freshly parsed collections when retrying failed items. + +Existing `backfill()` strategies control post extraction; later explicitly +configured strategies can still run. Set `contextDiscovery: "ancestors"` to +search `inReplyTo` ancestors when the seed has no usable collection, bounded by `maxDepth`. The nearest usable collection wins, with reply-target property order breaking ties; a valid empty collection stays selected. Source selection applies to each ancestor. Discovery shares traversal limits, caching, cancellation, and ID deduplication with configured strategies. Per-object loader and verifier defaults remain effective across strategy orders, without -hydrating the seed or loaded ancestors. Collection -items retain origin `"collection"` and depth 0; ancestors are yielded only -with `"reply-tree"`. Without this option, context strategies make no ancestor -requests. These options do not unwrap -`Add` activities or replay updates and deletions. Histories containing only -`Add(Create(...))` activities yield nothing and do not trigger `context` -fallback. +hydrating the seed or loaded ancestors. Collection items retain origin +`"collection"` and depth 0; ancestors are yielded only with `"reply-tree"`. +Without this option, context strategies make no ancestor requests. For +`backfill()`, these options do not unwrap `Add` activities or replay updates +and deletions. Its histories containing only `Add(Create(...))` activities +yield nothing and do not trigger `context` fallback; `backfillActivities()` +yields those wrapper activities intact. Since Fedify 2.5.0, pass `dereference: false` to vocabulary accessors to read embedded or cached objects without fetching references. Singular accessors diff --git a/docs/manual/backfill.md b/docs/manual/backfill.md index 7879ebcb9..69809f936 100644 --- a/docs/manual/backfill.md +++ b/docs/manual/backfill.md @@ -12,7 +12,8 @@ Conversation backfill Fedify provides the *@fedify/backfill* package for reconstructing ActivityPub conversations that may be incomplete on the local server. It can retrieve post-like objects from [FEP-f228] context collections and optionally crawl -`inReplyTo` ancestors and `replies` descendants. +`inReplyTo` ancestors and `replies` descendants. Choose `backfill()` to retrieve +posts or `backfillActivities()` to retrieve original conversation activities. [FEP-f228]: https://w3id.org/fep/f228 @@ -119,7 +120,7 @@ is empty, has no supported items, or later item or page loads fail. Backfill does not try later history values or merge history with `context`. The selected source is reused across context strategies within the same traversal. -Backfill does not unwrap `Add` activities. Histories containing only +`backfill()` does not unwrap `Add` activities. Histories containing only `Add(Create(...))` activities yield nothing and do not trigger `context` fallback. @@ -141,6 +142,9 @@ By itself, it does not discover history on ancestors. It does not unwrap `Add` activities or replay updates and deletions. Omitting the option preserves existing source selection. +These collection selection rules also apply to `backfillActivities()`. Source +selection does not change which item types each function yields. + ### Discovering collections through ancestors *This option is available since Fedify 2.5.0.* @@ -196,13 +200,90 @@ memory use and copying time independently of `maxDepth`. Objects yielded by `reply-tree` may be copies rather than the caller's original instances. +Reading original activities +--------------------------- + +*This API is available since Fedify 2.5.0.* + +Use `backfillActivities()` to read original activities, including `Create`, +`Update`, `Delete`, and `Like`, instead of extracting posts. Choose this +function directly when restoring conversation state; it does not require +first calling `backfill()` or reading the collection twice: + +~~~~ typescript twoslash +import { backfillActivities, type BackfillContext } from "@fedify/backfill"; +import { Delete, Note } from "@fedify/vocab"; + +declare const context: BackfillContext; +declare const seed: Note; +// ---cut-before--- +for await (const item of backfillActivities(context, seed, { + contextSource: "history-first", + maxItems: 100, + maxRequests: 200, +})) { + const activity = item.object; + console.log(item.id, activity.actorId); + if (activity instanceof Delete) { + console.log(activity.objectId); // Reads the reference without fetching it. + } +} +~~~~ + +The result is `BackfillItem` regardless of the seed's type. The +stream preserves each activity's type, actor, ID, and object reference. It +does not require target dereferencing: a deleted post does not need to resolve +for its `Delete` activity to be yielded. Direct posts in mixed collections +are skipped, and wrapper activities such as `Add` are yielded intact. +A collection containing only posts produces an empty activity stream, which +does not mean the conversation is empty. + +Activities follow collection traversal order, including pagination through +`first` and `next`, without local sorting. Results are not necessarily +latest-first, and unordered collections provide no chronological guarantee. + +| Item field | Activity output | +| ---------- | ------------------------------------------------------- | +| `object` | The original activity, including `Create` itself | +| `id` | The activity's ID when present, not its target's ID | +| `strategy` | `"context-activities"` | +| `origin` | `"collection"` | +| `depth` | `0`, including collections discovered through ancestors | + +Deduplication uses activity IDs independently of post IDs. Distinct `Create`, +`Update`, and `Delete` activities for the same post are retained; repeating +an activity ID yields it once. Anonymous activities are yielded per occurrence. +`maxItems` counts yielded activities; duplicates and failed or unresolved +item loads do not consume that limit. + +`BackfillActivityOptions` select existing collection and traversal controls: +`contextSource`, `contextDiscovery`, `maxDepth`, `maxItems`, `maxRequests`, +`interval`, and `signal`. Source selection defaults to `context`; opt into +`contextSource: "history-first"` for the history selection and fallback rules +[described above](#selecting-conversation-history). Set +`contextDiscovery: "ancestors"` to search ancestors for a collection, bounded +by `maxDepth`. Ancestor discovery does not yield ancestor posts. The options +do not expose post extraction strategies or an `output` mode, and the activity +function does not perform reply-tree output. + +The request budget, interval, cancellation, cache, and failure rules below +apply to both APIs. Each invocation owns its budget and cache. Activity +discovery is not verified inbox delivery: applications choose which activities +to apply and perform their own authorization checks before changing local +state. This function does not replay activities or authorize them for you. + +Existing `backfill()` behavior remains unchanged. In particular, its +`context-activities` strategy still extracts posts from supported `Create` +activities and skips other activity types. + + Strategies ---------- -Strategies run in the configured order. They share request and item budgets, -abort state, document caching, and object ID deduplication. If multiple -strategies discover the same object, the first one keeps its `BackfillItem` -metadata. +`backfill()` strategies run in the configured order. They share request and +item budgets, abort state, document caching, and object ID deduplication. If +multiple strategies discover the same object, the first one keeps its +`BackfillItem` metadata. `"context-auto"` : Handles both direct post-like objects and supported `Create` activities @@ -279,7 +360,8 @@ Traversal controls ------------------ `maxItems` -: Limits the number of yielded objects. Skipped duplicates do not count. +: Limits yielded posts for `backfill()` or yielded activities for + `backfillActivities()`. Duplicates and skipped loads do not count. `maxRequests` : Limits calls to `documentLoader`. Embedded objects and collections accepted @@ -290,6 +372,7 @@ Traversal controls to 10. Immediate parents and direct replies have depth 1; their next-level parents or replies have depth 2, and so on. Context collection items have depth 0 and are not limited by this option. + For `backfillActivities()`, this option bounds only ancestor discovery. `interval` : Adds a delay between `documentLoader` requests. A callback receives the @@ -305,7 +388,8 @@ Traversal controls Caching and failures -------------------- -Dereferenced documents are cached in memory for one `backfill()` traversal. +Dereferenced documents are cached in memory for one invocation of either +`backfill()` or `backfillActivities()`. Applications that need persistent or shared caching can implement it in the provided `documentLoader`. @@ -314,6 +398,15 @@ be discovered. Thrown loader errors are not retained in the traversal cache, allowing the same IRI to be retried if another traversal path reaches it. A returned `null` is cached for the rest of the traversal. Aborting the provided signal stops traversal instead of skipping the request. + +Vocabulary collection objects also cache resolved items independently of the +traversal cache. If the loader returns a previously traversed collection +instance, its cached items and skipped-load placeholders can survive into a +later invocation. A failed item may therefore remain skipped even after it +becomes available. Return a freshly parsed collection when retrying failed +item loads through that collection; starting another backfill call alone +does not reset the collection object's item cache. This applies to both APIs. + Errors cloning embedded history and errors in interval configuration during collection source resolution also propagate. Neither these errors nor cancellation trigger source fallback or allow later strategies to continue. diff --git a/packages/backfill/README.md b/packages/backfill/README.md index 580b208cd..0dd7ee189 100644 --- a/packages/backfill/README.md +++ b/packages/backfill/README.md @@ -11,8 +11,10 @@ This package provides ActivityPub conversation backfill support for the [Fedify] ecosystem. It can retrieve post-like objects from a seed object's context collection, following the direct [FEP-f228] path where the context dereferences to a `Collection`, `OrderedCollection`, `CollectionPage`, -or `OrderedCollectionPage`. It can also use an opt-in reply-tree strategy to -walk `inReplyTo` ancestors and `replies` descendants when context collections +or `OrderedCollectionPage`. Use `backfill()` for posts or +`backfillActivities()` for original conversation activities. The post API can +also use an opt-in reply-tree strategy to walk `inReplyTo` ancestors and +`replies` descendants when context collections are unavailable or incomplete. [npm badge]: https://img.shields.io/npm/v/@fedify/backfill?logo=npm @@ -93,8 +95,8 @@ for await ( } ~~~~ -To read only FEP-f228 activity collections, enable the `context-activities` -strategy: +To extract posts only from supported `Create` activities in a context +collection, enable the `context-activities` strategy: ~~~~ typescript for await ( @@ -130,6 +132,50 @@ depth 1, their next-level parents or replies have depth 2, and so on. Reply-tree traversal defaults to a maximum depth of 10; set `maxDepth` to use a different limit. +### Reading original activities + +*This API is available since Fedify 2.5.0.* + +Use `backfillActivities()` when your application needs activities such as +`Create`, `Update`, `Delete`, or `Like` rather than extracted posts: + +~~~~ typescript +import { backfillActivities } from "@fedify/backfill"; + +for await (const item of backfillActivities({ documentLoader }, note, { + contextSource: "history-first", + maxItems: 100, + maxRequests: 200, +})) { + console.log(item.id, item.object.actorId, item.object.objectId); +} +~~~~ + +The function returns `BackfillItem` independently of the seed's +type. It preserves activities and their object references without requiring +target dereferencing, so a `Delete` can be yielded when its target no longer +resolves. It skips direct posts and yields wrapper activities such as `Add` +intact. Collection order is preserved across pages without sorting; results +are not necessarily latest-first. + +Activity IDs are deduplicated independently of post IDs. Anonymous activities +are yielded per occurrence. `maxItems` counts yielded activities, excluding +duplicates and skipped loads. Each item has the activity's `id`, +`strategy: "context-activities"`, `origin: "collection"`, and `depth: 0`. + +`BackfillActivityOptions` reuse collection selection and traversal controls: +`contextSource`, `contextDiscovery`, `maxDepth`, `maxItems`, `maxRequests`, +`interval`, and `signal`. Ancestor discovery can locate a collection; +`maxDepth` bounds that search. Activity traversal has no `strategies` or +`output` option and does not yield reply-tree posts. Each call owns its cache +and budget; callers can choose the activity stream directly without first +running post backfill. + +Collection discovery is not verified inbox delivery. Applications choose +which activities to apply and must perform their own authorization checks. +Existing `backfill()` calls retain their behavior, including the +`context-activities` strategy that extracts posts from `Create`. + ### Discovering collections through ancestors *This option is available since Fedify 2.5.0.* @@ -146,14 +192,16 @@ example and selection rules. Traversal controls ------------------ -All configured strategies share the same traversal controls: +Both functions use the following traversal controls. Within `backfill()`, +all configured strategies share them: - - `maxItems` limits the number of yielded objects. Skipped duplicates do - not count. + - `maxItems` limits yielded posts for `backfill()` or yielded activities for + `backfillActivities()`. Duplicates and skipped loads do not count. - `maxRequests` limits calls to `documentLoader`. Embedded objects and collections, including embedded pages, do not count. - `maxDepth` limits reply-tree traversal and ancestor collection discovery, - and defaults to 10. It does not limit context collection items. + and defaults to 10. For `backfillActivities()`, it bounds only ancestor + discovery. It does not limit context collection items. - `interval` adds a delay between loader requests. Its callback receives the zero-based request index. - `signal` cancels traversal and is forwarded to `documentLoader`. @@ -161,15 +209,21 @@ All configured strategies share the same traversal controls: An `interval` string requires the global `Temporal` API or a polyfill. `Temporal.DurationLike` objects work without the global API. -Without history selection or ancestor discovery, if the seed has no context, -or its context resolves to a non-collection, context strategies yield nothing. -Loader failures are skipped unless traversal is aborted. If the context -collection cannot be loaded, later -configured strategies can still run. Failed loads consume a request, and -later strategies can use embedded data even when the request budget is -exhausted. Configuration errors, such as an invalid `interval`, still -propagate. - -Dereferenced documents are cached in memory for one `backfill()` traversal. -Applications that need persistent or shared caching can provide it through -the `documentLoader`. +Without history selection or ancestor discovery, a missing context or a +context resolving to a non-collection produces no collection items in either +API. Loader failures are skipped unless traversal is aborted. In `backfill()`, +later configured strategies can still run if the collection cannot be loaded, +and can use embedded data even when the request budget is exhausted. Failed +loads consume a request. Configuration errors, such as an invalid `interval`, +still propagate. + +Dereferenced documents are cached in memory for one invocation of either +backfill function. Thrown loader errors are not cached; returned `null` values +are cached for that invocation. Applications that need persistent or shared +caching can provide it through the `documentLoader`. + +The traversal cache is separate from item caches on vocabulary collection +objects. If your loader returns the same previously traversed collection +instance, its cached items and skipped-load placeholders can survive across +calls. Return a freshly parsed collection when retrying failed item loads +through that collection. diff --git a/packages/backfill/src/backfill-activities.test.ts b/packages/backfill/src/backfill-activities.test.ts new file mode 100644 index 000000000..509891b38 --- /dev/null +++ b/packages/backfill/src/backfill-activities.test.ts @@ -0,0 +1,221 @@ +import { deepStrictEqual, rejects, strictEqual } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + Activity, + Create, + Delete, + Like, + Note, + OrderedCollection, + Update, +} from "@fedify/vocab"; +import { + backfill, + backfillActivities, + type BackfillActivityOptions, + type BackfillContext, + type BackfillItem, +} from "./mod.ts"; + +describe("backfillActivities", () => { + it("does not expose skipped items cached by an earlier post traversal", async () => { + const collectionId = new URL("https://example.com/context"); + const missingId = new URL("https://example.com/missing"); + const collection = new OrderedCollection({ + items: [missingId, new Activity({})], + }); + const context: BackfillContext = { + documentLoader: (iri) => + Promise.resolve(iri.href === collectionId.href ? collection : null), + }; + const seed = new Note({ contexts: [collectionId] }); + deepStrictEqual(await Array.fromAsync(backfill(context, seed)), []); + const items = await Array.fromAsync(backfillActivities(context, seed)); + strictEqual(items.length, 1); + strictEqual(items[0].object.constructor, Activity); + }); + + it("lazily preserves activities, references, order, and activity metadata", async () => { + const collectionId = new URL("https://example.com/context"); + const seed = new Note({ + id: new URL("https://example.com/post"), + contexts: [collectionId], + }); + const actor = new URL("https://example.com/actor"); + const activities = [Create, Update, Delete, Like].map( + (Type, index) => + new Type({ + // Activity identity must be independent of seed/post identity. + id: + index === 0 + ? seed.id + : new URL(`https://example.com/activity/${index}`), + actor, + object: seed.id, + }), + ); + const requests: string[] = []; + const context: BackfillContext = { + documentLoader: (iri) => { + requests.push(iri.href); + strictEqual(iri.href, collectionId.href); + return Promise.resolve( + new OrderedCollection({ + items: [ + seed, + activities[0], + activities[0], + ...activities.slice(1), + new Like({ id: new URL("https://example.com/extra") }), + ], + }), + ); + }, + }; + const unsupported = { strategies: ["reply-tree"] as const }; + // @ts-expect-error Post strategies are excluded even through a variable. + const invalidStrategies: BackfillActivityOptions = unsupported; + const invalidOutput: BackfillActivityOptions = { + // @ts-expect-error There is no output option on the activity API. + output: "activities", + }; + // These declarations check types only; no runtime validation is required. + void invalidStrategies; + void invalidOutput; + // The Note seed does not determine the iterator's result type. + const stream: AsyncGenerator< + BackfillItem, + void, + void + > = backfillActivities(context, seed, { maxItems: activities.length }); + deepStrictEqual(requests, []); + const items = await Array.fromAsync(stream); + deepStrictEqual( + items.map((item) => item.object), + activities, + ); + deepStrictEqual(requests, [collectionId.href]); + for (const item of items) { + strictEqual(item.id?.href, item.object.id?.href); + strictEqual(item.object.actorId?.href, actor.href); + strictEqual(item.object.objectId?.href, seed.id?.href); + strictEqual(item.strategy, "context-activities"); + strictEqual(item.origin, "collection"); + strictEqual(item.depth, 0); + } + }); + + it("skips unresolved and rejected items while loading referenced activities", async () => { + const collectionId = new URL("https://example.com/context"); + const missingId = new URL("https://example.com/missing"); + const failedId = new URL("https://example.com/failed"); + const activityId = new URL("https://example.com/activity"); + const activity = new Delete({ + id: activityId, + object: new URL("https://example.com/deleted"), + }); + const context: BackfillContext = { + documentLoader: (iri) => { + if (iri.href === collectionId.href) { + return Promise.resolve( + new OrderedCollection({ + items: [missingId, failedId, activityId, new Like({})], + }), + ); + } + if (iri.href === failedId.href) + return Promise.reject(new Error("failed")); + return Promise.resolve(iri.href === activityId.href ? activity : null); + }, + }; + const items = await Array.fromAsync( + backfillActivities(context, new Note({ contexts: [collectionId] }), { + maxItems: 1, + }), + ); + strictEqual(items.length, 1); + strictEqual(items[0].object.constructor, Delete); + strictEqual(items[0].id?.href, activityId.href); + strictEqual(items[0].object.objectId?.href, activity.objectId?.href); + }); + + it("passes history and ancestor discovery options to collection selection", async () => { + const parentId = new URL("https://example.com/parent"); + const requests: string[] = []; + const context: BackfillContext = { + documentLoader: (iri) => { + requests.push(iri.href); + strictEqual(iri.href, parentId.href); + return Promise.resolve( + new Note({ + id: parentId, + contextHistory: new OrderedCollection({ items: [new Like({})] }), + }), + ); + }, + }; + const items = await Array.fromAsync( + backfillActivities(context, new Note({ replyTarget: parentId }), { + contextSource: "history-first", + contextDiscovery: "ancestors", + maxDepth: 1, + }), + ); + strictEqual(items.length, 1); + strictEqual(items[0].object.constructor, Like); + strictEqual(items[0].depth, 0); + deepStrictEqual(requests, [parentId.href]); + }); + + it("retains yielded activities when the request budget is exhausted", async () => { + const collectionId = new URL("https://example.com/context"); + const activityId = new URL("https://example.com/activity"); + const embedded = new Like({}); + const requests: string[] = []; + const context: BackfillContext = { + documentLoader: (iri) => { + requests.push(iri.href); + strictEqual(iri.href, collectionId.href); + return Promise.resolve( + new OrderedCollection({ items: [embedded, activityId] }), + ); + }, + }; + const items = await Array.fromAsync( + backfillActivities(context, new Note({ contexts: [collectionId] }), { + maxRequests: 1, + }), + ); + deepStrictEqual( + items.map((item) => item.object), + [embedded], + ); + deepStrictEqual(requests, [collectionId.href]); + }); + + it("propagates cancellation without yielding a loaded activity", async () => { + const controller = new AbortController(); + const reason = new Error("cancelled"); + const context: BackfillContext = { + documentLoader: (_iri, options) => { + strictEqual(options?.signal, controller.signal); + controller.abort(reason); + return Promise.resolve( + new OrderedCollection({ items: [new Delete({})] }), + ); + }, + }; + await rejects( + Array.fromAsync( + backfillActivities( + context, + new Note({ contexts: [new URL("https://example.com/context")] }), + { + signal: controller.signal, + }, + ), + ), + (error) => error === reason, + ); + }); +}); diff --git a/packages/backfill/src/backfill.ts b/packages/backfill/src/backfill.ts index e59462175..a310eab85 100644 --- a/packages/backfill/src/backfill.ts +++ b/packages/backfill/src/backfill.ts @@ -15,6 +15,7 @@ import { } from "@fedify/vocab-runtime/internal/portable-dereference"; import type { + BackfillActivityOptions, BackfillContext, BackfillItem, BackfillOptions, @@ -166,6 +167,80 @@ export async function* backfill( } } +/** + * Backfills original activities from a conversation collection in traversal + * order, without sorting or dereferencing activity targets. + * + * Uses the same collection selection and traversal controls as {@link backfill}. + * Direct posts are skipped. Activities are deduplicated by activity ID, + * independently of the seed post ID; anonymous activities are yielded per + * occurrence. `maxItems` counts yielded activities. + * + * Items reuse {@link BackfillItem}: `object` is the activity, `id` is its ID, + * `strategy` is `"context-activities"`, `origin` is `"collection"`, and + * `depth` is 0, even when ancestors were searched to discover the collection. + * Unlike the existing post strategy, this function preserves `Create` itself. + * Collection discovery does not constitute verified inbox delivery; callers + * must authorize activities before applying them. + * + * @since 2.5.0 + */ +export async function* backfillActivities( + context: BackfillContext, + seed: APObject, + options: BackfillActivityOptions = {}, +): AsyncGenerator, void, void> { + if (options.maxItems != null && options.maxItems <= 0) return; + const budget: RequestBudget = { + signal: options.signal, + requestCount: 0, + documents: new Map(), + discovery: + options.contextDiscovery === "ancestors" + ? { copies: new WeakMap(), sources: new WeakMap() } + : undefined, + }; + const seenIds = new Set(); + let yielded = 0; + try { + const collection = await discoverConversationCollection( + context, + copyDiscoveryObject(seed, budget), + options, + budget, + ); + if (collection == null) return; + for await (const object of getCollectionItems( + context, + collection, + options, + budget, + seenIds, + )) { + if (!(object instanceof Activity)) continue; + const id = object.id ?? undefined; + if (id != null) { + if (seenIds.has(id.href)) continue; + seenIds.add(id.href); + } + options.signal?.throwIfAborted(); + yield { + object, + id, + strategy: "context-activities", + origin: "collection", + depth: 0, + }; + yielded++; + if (options.maxItems != null && yielded >= options.maxItems) return; + } + } catch (error) { + if (error instanceof IntervalConfigurationError) throw error.cause; + if (error instanceof MaxRequestsExceeded) return; + throw error; + } +} + function normalizeStrategies( strategies: readonly BackfillStrategy[] = defaultStrategies, ): readonly BackfillStrategy[] { @@ -923,7 +998,10 @@ function skippedCollectionItemDocument(url: string) { documentUrl: url, document: { "@context": "https://www.w3.org/ns/activitystreams", - type: "Activity", + // An Activity placeholder could leak into backfillActivities() output as + // a real activity. Both APIs ignore Link placeholders, even when a + // collection caches one and is reused by a different traversal. + type: "Link", }, }; } diff --git a/packages/backfill/src/mod.ts b/packages/backfill/src/mod.ts index d72c3c339..1bc350466 100644 --- a/packages/backfill/src/mod.ts +++ b/packages/backfill/src/mod.ts @@ -6,8 +6,13 @@ * * @module */ -export { backfill, MaxRequestsExceeded } from "./backfill.ts"; +export { + backfill, + backfillActivities, + MaxRequestsExceeded, +} from "./backfill.ts"; export type { + BackfillActivityOptions, BackfillContext, BackfillDocumentLoader, BackfillDocumentLoaderOptions, diff --git a/packages/backfill/src/types.ts b/packages/backfill/src/types.ts index 14d349798..aa0769deb 100644 --- a/packages/backfill/src/types.ts +++ b/packages/backfill/src/types.ts @@ -170,6 +170,33 @@ export interface BackfillOptions< readonly signal?: AbortSignal; } +/** + * Controls {@link backfillActivities} collection traversal. + * + * Reuses post backfill's collection source selection, ancestor discovery, + * request limits, caching, intervals, and cancellation. `maxItems` counts + * yielded activities; `maxDepth` bounds ancestor discovery only. Reply-tree + * traversal and post extraction strategies are not supported. + * + * @since 2.5.0 + */ +export interface BackfillActivityOptions extends Pick< + BackfillOptions, + | "contextSource" + | "contextDiscovery" + | "maxDepth" + | "maxItems" + | "maxRequests" + | "interval" + | "signal" +> { + /** + * Activity traversal always preserves collection activities and does not + * accept post backfill strategies. + */ + readonly strategies?: never; +} + /** * A single object discovered by backfill traversal. *