Add a Netlify Blobs-backed KvStore

CHANGES.md

@@ -250,6 +250,10 @@ To be released.
 
 ### @fedify/netlify
 
+ -  Added `NetlifyBlobsKvStore`, a Netlify Blobs-backed key–value store with
+    expiration, prefix listing, and atomic compare-and-set operations.  Netlify
+    deployments can now persist Fedify state and preserve ordered queue
+    delivery without a separate database.  [[#1010], [#1029] by Jiwon Kwon\]
  -  Added the new *@fedify/netlify* package for processing Fedify message queue
     jobs with Netlify Async Workloads.  It provides `NetlifyMessageQueue` for
     durable event submission and `createNetlifyQueueHandler()` for Netlify
@@ -258,6 +262,9 @@ To be released.
     explicit recovery for unobservable dead-letter failures.
     [[#930], [#934]]
 
+[#1010]: https://github.com/fedify-dev/fedify/issues/1010
+[#1029]: https://github.com/fedify-dev/fedify/pull/1029
+
 ### @fedify/pglite
 
  -  Added the `@fedify/pglite` package with `PgliteKvStore`, a `KvStore` backed

changes.d/netlify/blobs-kv-store.md

@@ -0,0 +1,8 @@
+---
+links:
+  '#1029': https://github.com/fedify-dev/fedify/pull/1029
+---
+ -  Added `NetlifyBlobsKvStore`, a Netlify Blobs-backed key–value store with
+    expiration, prefix listing, and atomic compare-and-set operations.  Netlify
+    deployments can now persist Fedify state and preserve ordered queue
+    delivery without a separate database.  [[#1010], [#1029] by Jiwon Kwon]

docs/manual/kv.md

@@ -511,6 +511,79 @@ export default {
 [Cloudflare Workers]: https://workers.cloudflare.com/
 [Cloudflare Workers KV]: https://developers.cloudflare.com/kv/
 
+### `NetlifyBlobsKvStore`
+
+*This API is available since Fedify 2.4.0.*
+
+To use the [`NetlifyBlobsKvStore`], you need to install *@fedify/netlify* and
+*@netlify/blobs*.
+
+::: code-group
+
+~~~~ bash [Deno]
+deno add jsr:@fedify/netlify npm:@netlify/blobs
+~~~~
+
+~~~~ bash [npm]
+npm add @fedify/netlify @netlify/blobs
+~~~~
+
+~~~~ bash [pnpm]
+pnpm add @fedify/netlify @netlify/blobs
+~~~~
+
+~~~~ bash [Yarn]
+yarn add @fedify/netlify @netlify/blobs
+~~~~
+
+~~~~ bash [Bun]
+bun add @fedify/netlify @netlify/blobs
+~~~~
+
+:::
+
+`NetlifyBlobsKvStore` from `@fedify/netlify` stores federation state in
+[Netlify Blobs]
+without requiring a separate database.  It supports expiration, prefix
+listing, and atomic compare-and-set (CAS) operations.
+
+Best for
+:   Small Fedify applications on [Netlify Functions].
+
+Pros
+:   Persistent storage and atomic CAS without a separate database.
+
+Cons
+:   600-byte encoded key limit; expired blobs and tombstones are not
+    automatically removed.
+
+~~~~ typescript
+import { createFederation } from "@fedify/fedify";
+import { NetlifyBlobsKvStore } from "@fedify/netlify";
+import { getStore } from "@netlify/blobs";
+
+const federation = createFederation<void>({
+  kv: new NetlifyBlobsKvStore(getStore({
+    name: "fedify",
+    consistency: "strong",
+  })),
+});
+~~~~
+
+CAS operations always use strong reads and conditional writes.  Configuring
+the Blobs store with `consistency: "strong"` also enables strong consistency
+for ordinary reads and listings.
+
+> [!NOTE]
+> If your Netlify application already uses PostgreSQL, you can use
+> [`PostgresKvStore`](#postgreskvstore) from `@fedify/postgres` instead.
+> Netlify Blobs is optional; choose the store that matches your application's
+> storage setup.
+
+[`NetlifyBlobsKvStore`]: https://jsr.io/@fedify/netlify/doc/~/NetlifyBlobsKvStore
+[Netlify Blobs]: https://docs.netlify.com/build/data-and-storage/netlify-blobs/
+[Netlify Functions]: https://docs.netlify.com/build/functions/overview/
+
 
 Implementing a custom `KvStore`
 -------------------------------

package.json

@@ -1,5 +1,13 @@
 {
   "scripts": {
     "prepare": "pnpm -r run build:self"
+  },
+  "pnpm": {
+    "onlyBuiltDependencies": [
+      "esbuild"
+    ],
+    "patchedDependencies": {
+      "@netlify/blobs@11.0.3": "patches/@netlify__blobs@11.0.3.patch"
+    }
   }
 }

packages/netlify/README.md

@@ -1,18 +1,19 @@
 <!-- deno-fmt-ignore-file -->
 
-@fedify/netlify: Run Fedify queues with Netlify Async Workloads
-===============================================================
+@fedify/netlify: Run Fedify with Netlify blobs and async workloads
+==================================================================
 
 [![JSR][JSR badge]][JSR]
 [![npm][npm badge]][npm]
 [![@fedify@hackers.pub][@fedify@hackers.pub badge]][@fedify@hackers.pub]
 
 *This package is available since Fedify 2.4.0.*
 
-This package connects [Fedify]'s [`MessageQueue`] API to
-[Netlify Async Workloads].  `NetlifyMessageQueue` publishes durable events,
-while `createNetlifyQueueHandler()` turns a Netlify Function into their
-consumer.
+This package connects [Fedify]'s [`KvStore`] and [`MessageQueue`] APIs to
+Netlify.  `NetlifyBlobsKvStore` stores federation state in [Netlify Blobs],
+`NetlifyMessageQueue` publishes durable events through
+[Netlify Async Workloads], and `createNetlifyQueueHandler()` turns a Netlify
+Function into their consumer.
 
 The initial release targets Netlify Functions, not Netlify Edge Functions.
 
@@ -23,13 +24,17 @@ The initial release targets Netlify Functions, not Netlify Edge Functions.
 [@fedify@hackers.pub badge]: https://fedi-badge.minhee.org/@fedify@hackers.pub/followers.svg
 [@fedify@hackers.pub]: https://hackers.pub/@fedify
 [Fedify]: https://fedify.dev/
+[`KvStore`]: https://jsr.io/@fedify/fedify/doc/federation/~/KvStore
 [`MessageQueue`]: https://jsr.io/@fedify/fedify/doc/federation/~/MessageQueue
+[Netlify Blobs]: https://docs.netlify.com/build/data-and-storage/netlify-blobs/
 [Netlify Async Workloads]: https://docs.netlify.com/build/async-workloads/get-started/
 
 
 Installation
 ------------
 
+For queue-only usage, install the package with Netlify Async Workloads:
+
 ~~~~ sh
 deno add jsr:@fedify/netlify npm:@netlify/async-workloads  # Deno
 npm  add     @fedify/netlify @netlify/async-workloads     # npm
@@ -38,8 +43,62 @@ yarn add     @fedify/netlify @netlify/async-workloads     # Yarn
 bun  add     @fedify/netlify @netlify/async-workloads     # Bun
 ~~~~
 
-The PostgreSQL-backed `orderingKv` example below also needs
-`@fedify/postgres`, `@netlify/database`, and `postgres`:
+
+Usage
+-----
+
+To use `NetlifyBlobsKvStore`, also install Netlify Blobs:
+
+~~~~ sh
+deno add npm:@netlify/blobs  # Deno
+npm  add     @netlify/blobs  # npm
+pnpm add     @netlify/blobs  # pnpm
+yarn add     @netlify/blobs  # Yarn
+bun  add     @netlify/blobs  # Bun
+~~~~
+
+Create one store and queue for both the web application and the workload
+function.  `NetlifyBlobsKvStore` provides the compare-and-set (CAS) support
+required when Fedify emits messages with an `orderingKey`, without requiring a
+separate database:
+
+~~~~ typescript
+import { AsyncWorkloadsClient } from "@netlify/async-workloads";
+import { getStore } from "@netlify/blobs";
+import {
+  NetlifyBlobsKvStore,
+  NetlifyMessageQueue,
+} from "@fedify/netlify";
+
+export const kv = new NetlifyBlobsKvStore(getStore({
+  name: "fedify",
+  consistency: "strong",
+}));
+export const queue = new NetlifyMessageQueue({
+  client: new AsyncWorkloadsClient(),
+  orderingKv: kv,
+});
+~~~~
+
+CAS operations always use strong reads and conditional writes.  The
+`consistency: "strong"` setting above also gives ordinary `get()` calls and
+listings strong consistency.  You can omit it when lower-latency, eventually
+consistent ordinary reads are acceptable.
+
+TTL expiration is logical: expired blobs remain in Netlify Blobs but are
+hidden by `get()` and `list()`.  CAS deletion similarly writes a tombstone so
+that a concurrent writer cannot recreate the key undetected.  Tombstones are
+also hidden and can be replaced by a later CAS operation, but they are not
+removed automatically.  Use a dedicated Netlify Blobs store and account for
+these retained blobs.  Only remove them during maintenance when concurrent CAS
+operations for those keys have stopped; physical deletion during a race would
+remove the version token that protects the key.
+
+### PostgreSQL alternative
+
+If the application already uses PostgreSQL, `PostgresKvStore` can provide the
+same CAS-capable ordering storage.  Install `@fedify/postgres`,
+`@netlify/database`, and `postgres` in addition to the base dependencies:
 
 ~~~~ sh
 deno add jsr:@fedify/postgres npm:@netlify/database npm:postgres  # Deno
@@ -49,29 +108,20 @@ yarn add     @fedify/postgres @netlify/database postgres          # Yarn
 bun  add     @fedify/postgres @netlify/database postgres          # Bun
 ~~~~
 
-
-Usage
------
-
-Create one queue for both the web application and the workload function.  A
-CAS-capable `KvStore`, such as `PostgresKvStore`, is required if Fedify emits
-messages with an `orderingKey`:
+Then replace the `kv` declaration from the main example:
 
 ~~~~ typescript
-import { AsyncWorkloadsClient } from "@netlify/async-workloads";
 import { getConnectionString } from "@netlify/database";
-import { NetlifyMessageQueue } from "@fedify/netlify";
 import { PostgresKvStore } from "@fedify/postgres";
 import postgres from "postgres";
 
 const sql = postgres(getConnectionString());
 export const kv = new PostgresKvStore(sql);
-export const queue = new NetlifyMessageQueue({
-  client: new AsyncWorkloadsClient(),
-  orderingKv: kv,
-});
 ~~~~
 
+Ordering state must use crash-safe storage.  `PostgresKvStore` creates a logged
+table by default; do not pass `unlogged: true` for `orderingKv`.
+
 Pass the queue to Fedify with `manuallyStartQueue: true`.  Async Workloads
 invokes the consumer, so `NetlifyMessageQueue.listen()` is intentionally not
 available:
@@ -142,9 +192,6 @@ because a lost response does not prove that the router rejected the event.
 Use the error's `orderingKey` and `orderingSequence` for manual recovery only
 after ruling out delivery.
 
-Ordering state must use crash-safe storage.  `PostgresKvStore` creates a logged
-table by default; do not pass `unlogged: true` for `orderingKv`.
-
 Netlify limits an event payload to 500 KB.  Fedify messages, including any
 embedded activity, must remain below that limit.
 
@@ -156,7 +203,7 @@ Fedify rejects a multi-message `enqueueTaskMany()` call with one
 batch enqueue so a failed partial send cannot produce duplicates on retry.
 
 See the [deployment manual] and the [Netlify Astro example] for a complete
-setup with Netlify Database.
+setup using the PostgreSQL alternative with Netlify Database.
 
 [deployment manual]: https://fedify.dev/manual/deploy#netlify-functions
 [Netlify Astro example]: https://github.com/fedify-dev/fedify/tree/main/examples/netlify-astro

packages/netlify/deno.json

@@ -3,9 +3,9 @@
   "version": "2.4.0",
   "license": "MIT",
   "imports": {
-    "@netlify/database": "npm:@netlify/database@^1.1.0",
     "@netlify/async-workloads": "npm:@netlify/async-workloads@^0.0.106",
-    "postgres": "npm:postgres@^3.4.7"
+    "@netlify/blobs": "npm:@netlify/blobs@^11.0.3",
+    "byte-encodings": "npm:byte-encodings@^1.0.11"
   },
   "exports": {
     ".": "./src/mod.ts"

packages/netlify/fixtures/netlify-dev/netlify/functions/blob-kv.mts

@@ -0,0 +1,48 @@
+import { createServices } from "../lib/runtime.ts";
+
+interface BlobKvBody {
+  readonly action?: unknown;
+  readonly expectedValue?: unknown;
+  readonly id?: unknown;
+  readonly newValue?: unknown;
+  readonly value?: unknown;
+}
+
+export default async function blobKv(request: Request): Promise<Response> {
+  const body = await request.json() as BlobKvBody;
+  if (
+    typeof body.id !== "string" || body.id.length < 1 ||
+    typeof body.action !== "string"
+  ) {
+    return new Response("Invalid payload.", { status: 400 });
+  }
+
+  const { kv } = createServices();
+  const key = ["integration", "blob-kv", body.id] as const;
+  switch (body.action) {
+    case "set":
+      if (!("value" in body)) {
+        return new Response("Missing value.", { status: 400 });
+      }
+      await kv.set(key, body.value);
+      return Response.json({ written: true });
+    case "get": {
+      const value = await kv.get(key);
+      return value === undefined
+        ? Response.json({ found: false })
+        : Response.json({ found: true, value });
+    }
+    case "delete":
+      await kv.delete(key);
+      return Response.json({ deleted: true });
+    case "cas":
+      if (kv.cas == null) {
+        return new Response("CAS is unavailable.", { status: 500 });
+      }
+      return Response.json({
+        swapped: await kv.cas(key, body.expectedValue, body.newValue),
+      });
+    default:
+      return new Response("Invalid action.", { status: 400 });
+  }
+}

packages/netlify/fixtures/netlify-dev/netlify/lib/runtime.ts

@@ -4,12 +4,13 @@ import {
   type KvKey,
   type KvStore,
 } from "@fedify/fedify/federation";
-import { NetlifyMessageQueue } from "../../../../src/mod.ts";
-import { PostgresKvStore } from "@fedify/postgres";
+import {
+  NetlifyBlobsKvStore,
+  NetlifyMessageQueue,
+} from "../../../../src/mod.ts";
 import { AsyncWorkloadsClient } from "@netlify/async-workloads";
-import { getConnectionString } from "@netlify/database";
+import { getStore } from "@netlify/blobs";
 import type { StandardSchemaV1 } from "@standard-schema/spec";
-import postgres from "postgres";
 
 export interface TaskPayload {
   readonly id: string;
@@ -46,10 +47,10 @@ const taskSchema: StandardSchemaV1<unknown, TaskPayload> = {
 };
 
 export function createServices() {
-  const sql = postgres(getConnectionString());
-  const kv = new PostgresKvStore(sql, {
-    tableName: "fedify_netlify_integration_kv",
-  });
+  const kv = new NetlifyBlobsKvStore(getStore({
+    name: "fedify-integration",
+    consistency: "strong",
+  }));
   const baseUrl = process.env.URL ?? process.env.DEPLOY_URL;
   const queue = new NetlifyMessageQueue({
     client: new AsyncWorkloadsClient(baseUrl == null ? undefined : { baseUrl }),

packages/netlify/fixtures/public-types.ts

@@ -0,0 +1,7 @@
+import { getStore } from "@netlify/blobs";
+import { NetlifyBlobsKvStore } from "../dist/mod.js";
+
+new NetlifyBlobsKvStore(getStore({
+  name: "fedify",
+  consistency: "strong",
+}));