feat(media): moderate every uploaded image and video

.env.example

@@ -53,6 +53,30 @@ REDIS_URL=
 #DEEP_API
 DEEPL_API_KEY=dumplicate_deepl_api_key
 
+# --- Media moderation ---
+# Every upload endpoint - post media, comment media, article covers, avatars
+# and banners - is scanned for sexual, violent and gore content. Images are
+# checked inside the upload request and never reach storage if refused; videos
+# are stored, withheld from the read path, and checked by a background worker.
+# Leave MODERATION_ENABLED=false locally and in tests: a stand-in then approves
+# everything so no API key is needed to upload a picture.
+MODERATION_ENABLED=true
+SIGHTENGINE_API_USER=
+SIGHTENGINE_API_SECRET=
+# At or above the reject threshold an explicit class refuses the upload; at or
+# above the sensitive one the media is served but flagged for the client to
+# blur. Raw scores are stored on every asset so these can be retuned.
+MODERATION_REJECT_THRESHOLD=0.75
+MODERATION_SENSITIVE_THRESHOLD=0.4
+MODERATION_REQUEST_TIMEOUT_MS=15000
+# The video worker. Runs every minute: a user is waiting to see their own post.
+MEDIA_MODERATION_CRON="* * * * *"
+MEDIA_MODERATION_BATCH_SIZE=10
+MEDIA_MODERATION_MAX_ATTEMPTS=3
+# How long a worker's claim is honoured. A process killed mid-scan leaves the
+# asset claimed; without this the post carrying it would hide its media forever.
+MEDIA_MODERATION_LEASE_SECONDS=600
+
 # --- Feed ranking ---
 # Weights the feed ranker mixes. A post in the viewer's language starts at 4x
 # the score of one they cannot read; roughly two half-lives of extra freshness
@@ -64,7 +88,7 @@ FEED_HALF_LIFE_HOURS=18
 # Most posts one author may hold in the ranked head.
 FEED_MAX_POSTS_PER_AUTHOR=3
 # Largest share of the feed that may be in a language the viewer does not read.
-FEED_FOREIGN_LANGUAGE_QUOTA=0.25
+FEED_FOREIGN_LANGUAGE_QUOTA=0.2
 # Size of the candidate pool the ranker scores, and how far back it reaches.
 FEED_CANDIDATE_POOL_SIZE=300
 FEED_CANDIDATE_WINDOW_DAYS=7

CLAUDE.md

@@ -75,6 +75,19 @@ Auth decorators: `fastify.authenticate` (required) and `fastify.optionalAuthenti
 
 `RateLimitPolicies` in `src/http/plugins/rate-limit.plugin.ts`: `STRICT` (3/15 min, `continueExceeding`) for login/register, `SENSITIVE` (5/min) for password reset, verification, and write/social actions, `STANDARD` (60/min) for authenticated reads, `PUBLIC` (100/min). Global default is 100/min. Requests with `Authorization: Bot <token>` are allow-listed after a sha256 lookup against `user.botToken`.
 
+### Media moderation
+
+Every upload endpoint — `POST /media` (post *and* comment media), `POST /articles/cover`, `PATCH /me/avatar`, `PATCH /me/banner` — goes through `UploadModeratedMediaUseCase` (`src/core/use-cases/media/upload-moderated-media/`). It sniffs the format from the file's magic bytes (`src/core/use-cases/shared/media/detect-media-type.ts`; the client's MIME type and filename are never trusted), then splits:
+
+- **Images** are scanned before a byte reaches R2, so a refused file never gets a URL. A provider error fails the upload closed (`ModerationUnavailableError`, 503).
+- **Videos** are stored as `PENDING` and scanned by the `media-moderation` cron worker, because the provider has to fetch and sample them.
+
+Each stored file gets a `MediaAsset` row. **That row is what makes an uploaded key trustworthy:** `CreatePostUseCase` and `CreateCommentUseCase` resolve every submitted `mediaUrls` entry back to an asset via `resolveAttachableMedia` and reject it (`MediaNotOwnedError`, 400) unless this author uploaded it, through the matching `MediaChannel`, and moderation did not reject it. Without that check the pipeline is decorative — the request body accepts arbitrary URLs.
+
+Verdicts are tiered: explicit sexual content, gore, self-harm and hate imagery are rejected; suggestive content, weapons and depicted violence only set `isSensitive` so the client blurs them (this platform is full of game screenshots). Thresholds and the class-to-tier map live in `src/infrastructure/external/moderation/score-to-verdict.ts`, and raw provider scores are stored on every asset so they can be retuned.
+
+Posts, comments and articles carry denormalised `isSensitive` / `mediaStatus` columns; the mappers withhold `mediaUrls` while `mediaStatus !== APPROVED` but still serve the text. `MODERATION_ENABLED=false` (test and local) swaps in `NoopModerationService`, which approves everything — it is never a fallback for a provider that is down.
+
 ### Realtime and background jobs
 
 `FastifyRealtimeService` publishes to the Redis `realtime_events` channel; each instance subscribes and fans out to locally connected sockets via `WebSocketManager` — so notifications work across multiple processes. Never write to sockets directly from a use-case; go through `RealtimePort`.

prisma/migrations/20260901000000_add_media_moderation/migration.sql

@@ -0,0 +1,105 @@
+-- Automated moderation for every uploaded image and video.
+--
+-- Two things happen here. `media_assets` records one row per stored file:
+-- who uploaded it, through which endpoint, and what the moderation provider
+-- said about it. Posts, comments and articles gain the two denormalised
+-- columns the read path needs so it can withhold media without joining.
+--
+-- The asset table is what makes an uploaded key trustworthy. Scanning at
+-- upload time only governs what the upload endpoint writes to storage;
+-- nothing stops a client from skipping that endpoint and putting its own URL
+-- straight into a post body. Content creation now resolves every submitted
+-- URL back to a row here and refuses it unless this uploader owns it and
+-- moderation did not reject it.
+--
+-- Adding columns with a constant default is metadata-only in Postgres 11+, so
+-- no table is rewritten. Existing rows become `APPROVED` and not sensitive,
+-- which is the correct reading of them: they predate the pipeline, carry no
+-- assets, and must not disappear from feeds because of it. Backfilling them
+-- for a retroactive scan is a separate, out-of-band job.
+
+-- CreateEnum
+CREATE TYPE "public"."MediaModerationStatus" AS ENUM ('PENDING', 'SCANNING', 'APPROVED', 'SENSITIVE', 'REJECTED');
+
+-- CreateEnum
+CREATE TYPE "public"."MediaKind" AS ENUM ('IMAGE', 'VIDEO');
+
+-- The upload endpoint a file came through, fixed when the bytes arrive. Posts
+-- and comments share one endpoint, which is why POST_MEDIA covers both and why
+-- "which of the two claimed it" is a separate column.
+-- CreateEnum
+CREATE TYPE "public"."MediaChannel" AS ENUM ('POST_MEDIA', 'ARTICLE_COVER', 'AVATAR', 'BANNER');
+
+-- CreateEnum
+CREATE TYPE "public"."MediaOwnerKind" AS ENUM ('POST', 'COMMENT', 'ARTICLE');
+
+-- CreateTable
+CREATE TABLE "public"."media_assets" (
+    "id" TEXT NOT NULL,
+    "storage_key" TEXT NOT NULL,
+    "kind" "public"."MediaKind" NOT NULL,
+    "mime_type" TEXT NOT NULL,
+    "byte_size" INTEGER NOT NULL,
+    "uploader_id" TEXT NOT NULL,
+    "channel" "public"."MediaChannel" NOT NULL,
+    "owner_id" TEXT,
+    "owner_kind" "public"."MediaOwnerKind",
+    "status" "public"."MediaModerationStatus" NOT NULL DEFAULT 'PENDING',
+    "categories" TEXT[] DEFAULT ARRAY[]::TEXT[],
+    "scores" JSONB,
+    "provider" TEXT,
+    "moderated_at" TIMESTAMP(3),
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "last_error" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "media_assets_pkey" PRIMARY KEY ("id")
+);
+
+-- One row per stored object, so a key can never resolve to two different
+-- uploaders or two different verdicts.
+-- CreateIndex
+CREATE UNIQUE INDEX "media_assets_storage_key_key" ON "public"."media_assets" ("storage_key");
+
+-- The worker's claim query is "oldest pending first", and it runs every
+-- minute against a table that is mostly settled rows.
+-- CreateIndex
+CREATE INDEX "media_assets_status_created_at_idx" ON "public"."media_assets" ("status", "created_at");
+
+-- CreateIndex
+CREATE INDEX "media_assets_uploader_id_idx" ON "public"."media_assets" ("uploader_id");
+
+-- Backs the worker rebuilding an owner's media list from its surviving assets.
+-- CreateIndex
+CREATE INDEX "media_assets_owner_kind_owner_id_idx" ON "public"."media_assets" ("owner_kind", "owner_id");
+
+-- Deleting a user takes their assets with them; the objects themselves are
+-- swept by the existing purge job.
+-- AddForeignKey
+ALTER TABLE "public"."media_assets" ADD CONSTRAINT "media_assets_uploader_id_fkey" FOREIGN KEY ("uploader_id") REFERENCES "public"."users"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AlterTable
+ALTER TABLE "public"."posts"
+    ADD COLUMN "is_sensitive" BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN "media_status" "public"."MediaModerationStatus" NOT NULL DEFAULT 'APPROVED';
+
+-- AlterTable
+ALTER TABLE "public"."comments"
+    ADD COLUMN "is_sensitive" BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN "media_status" "public"."MediaModerationStatus" NOT NULL DEFAULT 'APPROVED';
+
+-- AlterTable
+ALTER TABLE "public"."articles"
+    ADD COLUMN "is_sensitive" BOOLEAN NOT NULL DEFAULT false,
+    ADD COLUMN "media_status" "public"."MediaModerationStatus" NOT NULL DEFAULT 'APPROVED';
+
+-- Tells an author their upload was removed. Self-issued - it comes from the
+-- platform, and there is no system account to attribute it to - so the type is
+-- what carries the meaning, not the issuer.
+--
+-- Postgres allows ALTER TYPE ... ADD VALUE inside a transaction from version
+-- 12 on, provided the new value is not used in the same transaction. Nothing
+-- here writes a MEDIA_REJECTED row.
+-- AlterEnum
+ALTER TYPE "public"."NotificationType" ADD VALUE 'MEDIA_REJECTED';

prisma/models/article.prisma

@@ -16,6 +16,13 @@ model Article {
   coverImageKey String? @map("cover_image_key")
   coverImageAlt String? @map("cover_image_alt") @db.VarChar(160)
 
+  /// The cover is the only media an article carries, and it is always an
+  /// image, so it is scanned inside the upload request and never sits
+  /// PENDING. The fields exist so the read path can treat every content type
+  /// the same way.
+  isSensitive Boolean               @default(false) @map("is_sensitive")
+  mediaStatus MediaModerationStatus @default(APPROVED) @map("media_status")
+
   status             ArticleStatus @default(DRAFT)
   publishedAt        DateTime?     @map("published_at")
   readingTimeMinutes Int           @default(1) @map("reading_time_minutes")

prisma/models/media.prisma

@@ -0,0 +1,84 @@
+/// Moderation lifecycle of a stored file.
+///
+/// Images never sit in PENDING: they are scanned inside the upload request and
+/// are only written to storage once a verdict exists. Videos do, because the
+/// provider needs to fetch and sample them, which is far too slow to hold an
+/// HTTP request open for.
+enum MediaModerationStatus {
+  PENDING
+  SCANNING
+  APPROVED
+  SENSITIVE
+  REJECTED
+}
+
+enum MediaKind {
+  IMAGE
+  VIDEO
+}
+
+/// The upload endpoint a file came through. Chosen at upload time, so an avatar
+/// can never be attached to a post afterwards. Posts and comments share one
+/// endpoint, so the channel deliberately does not say which of the two claimed
+/// the file - `ownerId` does that, once something has.
+enum MediaChannel {
+  POST_MEDIA
+  ARTICLE_COVER
+  AVATAR
+  BANNER
+}
+
+/// Which table `ownerId` points into. Posts and comments share one upload
+/// endpoint, so the channel cannot say, and a verdict has to know where to be
+/// written back to.
+enum MediaOwnerKind {
+  POST
+  COMMENT
+  ARTICLE
+}
+
+/// One stored file and everything the moderation pipeline knows about it.
+///
+/// The row is what makes an uploaded key trustworthy: content creation looks a
+/// key up here and refuses it unless this uploader owns it and it passed
+/// moderation. Without that check the pipeline would be decorative, since a
+/// client can put any URL it likes in a post body.
+model MediaAsset {
+  id String @id @default(uuid())
+
+  /// The R2 object key, without any CDN prefix.
+  storageKey String    @unique @map("storage_key")
+  kind       MediaKind
+  /// Derived from the file's magic bytes, never from the client's claim.
+  mimeType   String    @map("mime_type")
+  byteSize   Int       @map("byte_size")
+
+  uploaderId String @map("uploader_id")
+  uploader   User   @relation(fields: [uploaderId], references: [id], onDelete: Cascade)
+
+  channel   MediaChannel    @map("channel")
+  /// The post or comment the asset ended up on. Null until the content that
+  /// uses it is created, which is what lets a purge job find abandoned uploads.
+  ownerId   String?         @map("owner_id")
+  ownerKind MediaOwnerKind? @map("owner_kind")
+
+  status     MediaModerationStatus @default(PENDING)
+  /// Flagged category labels, kept for auditing a rejection after the fact.
+  categories String[]              @default([])
+  /// Raw provider scores. Stored so thresholds can be retuned against real
+  /// traffic instead of guesses.
+  scores     Json?
+  provider   String?
+
+  moderatedAt DateTime? @map("moderated_at")
+  attempts    Int       @default(0)
+  lastError   String?   @map("last_error")
+
+  createdAt DateTime @default(now()) @map("created_at")
+  updatedAt DateTime @updatedAt @map("updated_at")
+
+  @@index([status, createdAt])
+  @@index([uploaderId])
+  @@index([ownerKind, ownerId])
+  @@map("media_assets")
+}

prisma/models/notification.prisma

@@ -6,6 +6,7 @@ enum NotificationType {
   COMMENT_LIKE
   COMMENT_REPLY
   QUOTE
+  MEDIA_REJECTED
 }
 
 model Notification {

prisma/models/post.prisma

@@ -26,6 +26,14 @@ model Post {
   /// rather than being pushed out of every feed.
   lang String? @db.VarChar(5)
 
+  /// True when moderation judged the media borderline rather than forbidden.
+  /// The post is served as normal; the client blurs the media behind a tap.
+  isSensitive Boolean               @default(false) @map("is_sensitive")
+  /// APPROVED for a text-only post. PENDING while an attached video is still
+  /// being scanned and REJECTED once one failed, both of which make the read
+  /// path withhold the media while leaving the text visible.
+  mediaStatus MediaModerationStatus @default(APPROVED) @map("media_status")
+
   authorId String @map("author_id")
   author   User   @relation(fields: [authorId], references: [id], onDelete: Cascade)
 
@@ -41,9 +49,9 @@ model Post {
   bookmarks     PostBookmark[]
   comments      Comment[]
   notifications Notification[]
-  commentCount Int            @default(0)
-  likeCount    Int            @default(0)
-  quoteCount   Int            @default(0)
+  commentCount  Int            @default(0)
+  likeCount     Int            @default(0)
+  quoteCount    Int            @default(0)
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @updatedAt @map("updated_at")
@@ -115,6 +123,11 @@ model Comment {
   content   String   @db.Text
   mediaUrls String[] @default([]) @map("media_urls")
 
+  /// Mirrors the same two fields on Post: a comment carries media from the
+  /// same upload endpoint, so it needs the same withholding rules.
+  isSensitive Boolean               @default(false) @map("is_sensitive")
+  mediaStatus MediaModerationStatus @default(APPROVED) @map("media_status")
+
   // Exactly one of postId / articleId is set, enforced by a CHECK constraint
   // added in the migration: Prisma cannot express one.
   postId    String? @map("post_id")
@@ -132,8 +145,8 @@ model Comment {
   likes         CommentLike[]
   bookmarks     CommentBookmark[]
   notifications Notification[]
-  likeCount  Int               @default(0) @map("like_count")
-  replyCount Int               @default(0) @map("reply_count")
+  likeCount     Int               @default(0) @map("like_count")
+  replyCount    Int               @default(0) @map("reply_count")
 
   createdAt DateTime @default(now()) @map("created_at")
   updatedAt DateTime @updatedAt @map("updated_at")

prisma/models/user.prisma

@@ -29,6 +29,7 @@ model User {
   comments         Comment[]
   commentBookmarks CommentBookmark[]
 
+  mediaAssets      MediaAsset[]
   articles         Article[]
   articleLikes     ArticleLike[]
   articleBookmarks ArticleBookmark[]

render.yaml

@@ -11,6 +11,12 @@ projects:
       repo: https://github.com/the-developer-network/tdn-api
       plan: free
       envVars:
+      - key: MODERATION_ENABLED
+        sync: false
+      - key: SIGHTENGINE_API_USER
+        sync: false
+      - key: SIGHTENGINE_API_SECRET
+        sync: false
       - key: OTP_EXPIRY_SECONDS
         sync: false
       - key: USER_PURGE_GRACE_PERIOD_DAYS

src/app.ts

@@ -25,6 +25,7 @@ import realtimeRoutes from "@routes/realtime.routes";
 import notificationRoutes from "@routes/notification.routes";
 import notificationPurgePlugin from "@plugins/custom/notification-purge.plugin";
 import userInterestRebuildPlugin from "@plugins/custom/user-interest-rebuild.plugin";
+import mediaModerationPlugin from "@plugins/custom/media-moderation.plugin";
 import { postRoutes } from "@routes/post/post.routes";
 import { commentRoutes } from "@routes/post/comment.routes";
 import { likeRoutes } from "@routes/post/like.routes";
@@ -105,6 +106,7 @@ export class App {
         this.server.register(userPurgePlugin);
         this.server.register(notificationPurgePlugin);
         this.server.register(userInterestRebuildPlugin);
+        this.server.register(mediaModerationPlugin);
     }
 
     /**