fix: skip an empty r/all head listing instead of crashing the bot

AGENTS.md

@@ -121,7 +121,8 @@ A banned target (a 404 whose body says `"reason": "banned"`, `SUBREDDIT_IS_BANNE
     `STREAM_READERS` threads, keeping only what r/all could show (`is_shown_on_r_all`: not private or on a user
     profile). `/api/info` comments have no `over_18` or `quarantine` field, so Phase 1 drops NSFW and
     quarantined ones by their submission (`phase1.is_hidden_from_r_all`, no extra request). The head poll
-    asks the listing for 25 comments and takes the newest, because `limit=1` sometimes returns none. Subreddits that opted out of r/all cannot be told apart, so their comments
+    asks the listing for 25 comments and takes the newest, because `limit=1` sometimes returns none. An empty
+    listing anyway only skips that poll (one WARNING per streak); the next poll catches up. Subreddits that opted out of r/all cannot be told apart, so their comments
     are read too. `STREAM_SOURCE=listing` instead reads
     `reddit.subreddit('all').stream.comments(skip_existing=True, pause_after=-1)` one request at a time, which
     sees only a fraction of r/all (section 9).
@@ -401,7 +402,8 @@ Store these in `.env` for local development or production:
    the scheduled run, the row is deleted afterwards whether or not it was crossposted, unless `DRY_RUN` is on,
    so repeated runs walk through the table oldest first. Exit code 1 means the table is empty.
    Under Docker Compose, inside the running bot container: `docker compose exec bot python -m racb --process-one`
-   (add `-e DRY_RUN=True` after `exec` for a dry run).
+   (add `-e DRY_RUN=True` after `exec` for a dry run). A restart of the bot container SIGKILLs it silently
+   (exit code 137); `docker compose run --rm bot python -m racb --process-one` runs it in its own container.
    `--process-one` starts nothing else (no stream, jobs or worker threads) and authenticates only
    `AutoCrosspostBot`, the one account Phase 3 uses, so the other 3 need no valid credentials for it. It exits
    with code 2 when `AUTO_CROSSPOST_BOT_ENABLED` is false. Every other invocation except `--help` first runs

README.md

@@ -85,6 +85,11 @@ docker compose exec bot python -m racb --process-one                 # crosspost
 docker compose exec -e DRY_RUN=True bot python -m racb --process-one # only log the crosspost
 ```
 
+`exec` runs inside the bot's container, so if the bot restarts (it crashed, or a deploy replaced it), Docker kills
+the run mid-way without a message and the exit code is 137. Check `docker compose logs --tail 80 bot` for why
+the bot stopped. `docker compose run --rm bot python -m racb --process-one` runs it in a container of its own
+instead, which a bot restart does not touch.
+
 From your own computer, through SSH to the VM (same command in bash and PowerShell; use your VM name and
 zone, see [`docs/logs.md`](./docs/logs.md)). `-T` is needed because the SSH command runs without a terminal:
 ```bash

changelog.d/+empty-head-listing.fixed.md

@@ -0,0 +1,5 @@
+**The bot no longer restarts when the r/all comment listing comes back empty.** Reddit sometimes returns no comments
+for the comment reader's head poll even when asked for 25. That raised an error that ended the process, so Docker
+restarted the container over and over (71 times on the VM on 2026-10-16), and every restart also killed a
+`docker compose exec bot python -m racb --process-one` run silently, with exit code 137. An empty listing now only
+skips that poll, logged once per streak; the next poll queues everything it missed, so no comment is skipped.

src/racb/comment_reader.py

@@ -102,10 +102,13 @@ def is_shown_on_r_all(comment):
 
 
 def fetch_head():
-    """Returns the number of the newest comment on r/all (one request, as /u/AutoCrosspostBot)."""
+    """Returns the number of the newest comment on r/all (one request, as /u/AutoCrosspostBot).
+
+    Returns None if the listing came back empty, which Reddit sometimes does even with HEAD_LISTING_LIMIT.
+    """
     newest = list(reddit.get_reddit_instance().subreddit('all').comments(limit=HEAD_LISTING_LIMIT))
     if not newest:
-        raise RuntimeError('The r/all comment listing returned no comments')
+        return None
     return max(comment_id_to_number(comment.id) for comment in newest)
 
 
@@ -138,6 +141,8 @@ def __init__(self, handle_comment, reader_count, metrics=None, handle_exception=
         self.in_flight = set()
         self.done_above = set()
         self.last_saved = time.monotonic()
+        # Head polls in a row that got an empty listing, so a long streak is logged once, not every poll
+        self.empty_head_polls = 0
         # Readers take turns between the accounts, so each account's rate limit carries a share
         self.threads = [
             threading.Thread(
@@ -176,6 +181,15 @@ def poll_head(self):
         head = self.fetch_head()
         if self.metrics:
             self.metrics.record_head_poll(time.monotonic() - started)
+        if head is None:
+            # Nothing is lost: the next poll queues every block up to the head it finds
+            self.empty_head_polls += 1
+            if self.empty_head_polls == 1:
+                logger.warning('The r/all comment listing returned no comments; trying again at the next poll')
+            return
+        if self.empty_head_polls:
+            logger.info(f'The r/all comment listing has comments again after {self.empty_head_polls} empty poll(s)')
+            self.empty_head_polls = 0
         with self._lock:
             if self.next_id is None:
                 self._choose_start(head)

tests/unit/test_comment_reader.py

@@ -134,12 +134,47 @@ def test_fetch_head_takes_the_newest_of_several_comments():
 
 
 @pytest.mark.unit
-def test_fetch_head_raises_on_an_empty_listing():
+def test_fetch_head_returns_none_on_an_empty_listing():
     instance = MagicMock()
     instance.subreddit.return_value.comments.return_value = iter([])
     with patch('racb.comment_reader.reddit.get_reddit_instance', return_value=instance):
-        with pytest.raises(RuntimeError):
-            comment_reader.fetch_head()
+        assert comment_reader.fetch_head() is None
+
+
+@pytest.mark.unit
+def test_an_empty_head_listing_is_skipped_and_the_next_poll_catches_up():
+    reader, _ = make_reader(head=1000)
+    reader.poll_head()
+    head = {'value': None}
+    reader.fetch_head = lambda: head['value']
+    reader.poll_head()
+    reader.poll_head()
+    assert reader.empty_head_polls == 2
+    assert queued_blocks(reader) == []
+    head['value'] = 1000 + 3 * BLOCK_SIZE
+    reader.poll_head()
+    assert reader.empty_head_polls == 0
+    assert queued_blocks(reader) == [1001, 1101, 1201]
+
+
+@pytest.mark.unit
+def test_run_survives_an_empty_head_listing():
+    errors = []
+    polls = []
+
+    def empty_then_stop():
+        polls.append(1)
+        if len(polls) == 2:
+            shutdown.request_stop()
+        return None
+
+    reader, _ = make_reader()
+    reader.handle_exception = lambda error: errors.append(error) or True
+    reader.fetch_head = empty_then_stop
+    with patch('racb.comment_reader.HEAD_POLL_SECONDS', 0):
+        reader.run()
+    assert polls == [1, 1]
+    assert errors == []
 
 
 @pytest.mark.unit