diff --git a/AGENTS.md b/AGENTS.md index 778bf75..7298426 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,7 +121,8 @@ A banned target (a 404 whose body says `"reason": "banned"`, `SUBREDDIT_IS_BANNE `STREAM_READERS` threads, keeping only what r/all could show (`is_shown_on_r_all`: not private or on a user profile). `/api/info` comments have no `over_18` or `quarantine` field, so Phase 1 drops NSFW and quarantined ones by their submission (`phase1.is_hidden_from_r_all`, no extra request). The head poll - asks the listing for 25 comments and takes the newest, because `limit=1` sometimes returns none. Subreddits that opted out of r/all cannot be told apart, so their comments + asks the listing for 25 comments and takes the newest, because `limit=1` sometimes returns none. An empty + listing anyway only skips that poll (one WARNING per streak); the next poll catches up. Subreddits that opted out of r/all cannot be told apart, so their comments are read too. `STREAM_SOURCE=listing` instead reads `reddit.subreddit('all').stream.comments(skip_existing=True, pause_after=-1)` one request at a time, which sees only a fraction of r/all (section 9). @@ -401,7 +402,8 @@ Store these in `.env` for local development or production: the scheduled run, the row is deleted afterwards whether or not it was crossposted, unless `DRY_RUN` is on, so repeated runs walk through the table oldest first. Exit code 1 means the table is empty. Under Docker Compose, inside the running bot container: `docker compose exec bot python -m racb --process-one` - (add `-e DRY_RUN=True` after `exec` for a dry run). + (add `-e DRY_RUN=True` after `exec` for a dry run). A restart of the bot container SIGKILLs it silently + (exit code 137); `docker compose run --rm bot python -m racb --process-one` runs it in its own container. `--process-one` starts nothing else (no stream, jobs or worker threads) and authenticates only `AutoCrosspostBot`, the one account Phase 3 uses, so the other 3 need no valid credentials for it. It exits with code 2 when `AUTO_CROSSPOST_BOT_ENABLED` is false. Every other invocation except `--help` first runs diff --git a/README.md b/README.md index 3477bce..44ef12d 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,11 @@ docker compose exec bot python -m racb --process-one # crosspost docker compose exec -e DRY_RUN=True bot python -m racb --process-one # only log the crosspost ``` +`exec` runs inside the bot's container, so if the bot restarts (it crashed, or a deploy replaced it), Docker kills +the run mid-way without a message and the exit code is 137. Check `docker compose logs --tail 80 bot` for why +the bot stopped. `docker compose run --rm bot python -m racb --process-one` runs it in a container of its own +instead, which a bot restart does not touch. + From your own computer, through SSH to the VM (same command in bash and PowerShell; use your VM name and zone, see [`docs/logs.md`](./docs/logs.md)). `-T` is needed because the SSH command runs without a terminal: ```bash diff --git a/changelog.d/+empty-head-listing.fixed.md b/changelog.d/+empty-head-listing.fixed.md new file mode 100644 index 0000000..72763d1 --- /dev/null +++ b/changelog.d/+empty-head-listing.fixed.md @@ -0,0 +1,5 @@ +**The bot no longer restarts when the r/all comment listing comes back empty.** Reddit sometimes returns no comments +for the comment reader's head poll even when asked for 25. That raised an error that ended the process, so Docker +restarted the container over and over (71 times on the VM on 2026-10-16), and every restart also killed a +`docker compose exec bot python -m racb --process-one` run silently, with exit code 137. An empty listing now only +skips that poll, logged once per streak; the next poll queues everything it missed, so no comment is skipped. diff --git a/src/racb/comment_reader.py b/src/racb/comment_reader.py index 8595acf..e7c1c21 100644 --- a/src/racb/comment_reader.py +++ b/src/racb/comment_reader.py @@ -102,10 +102,13 @@ def is_shown_on_r_all(comment): def fetch_head(): - """Returns the number of the newest comment on r/all (one request, as /u/AutoCrosspostBot).""" + """Returns the number of the newest comment on r/all (one request, as /u/AutoCrosspostBot). + + Returns None if the listing came back empty, which Reddit sometimes does even with HEAD_LISTING_LIMIT. + """ newest = list(reddit.get_reddit_instance().subreddit('all').comments(limit=HEAD_LISTING_LIMIT)) if not newest: - raise RuntimeError('The r/all comment listing returned no comments') + return None return max(comment_id_to_number(comment.id) for comment in newest) @@ -138,6 +141,8 @@ def __init__(self, handle_comment, reader_count, metrics=None, handle_exception= self.in_flight = set() self.done_above = set() self.last_saved = time.monotonic() + # Head polls in a row that got an empty listing, so a long streak is logged once, not every poll + self.empty_head_polls = 0 # Readers take turns between the accounts, so each account's rate limit carries a share self.threads = [ threading.Thread( @@ -176,6 +181,15 @@ def poll_head(self): head = self.fetch_head() if self.metrics: self.metrics.record_head_poll(time.monotonic() - started) + if head is None: + # Nothing is lost: the next poll queues every block up to the head it finds + self.empty_head_polls += 1 + if self.empty_head_polls == 1: + logger.warning('The r/all comment listing returned no comments; trying again at the next poll') + return + if self.empty_head_polls: + logger.info(f'The r/all comment listing has comments again after {self.empty_head_polls} empty poll(s)') + self.empty_head_polls = 0 with self._lock: if self.next_id is None: self._choose_start(head) diff --git a/tests/unit/test_comment_reader.py b/tests/unit/test_comment_reader.py index 8773c97..f18a63f 100644 --- a/tests/unit/test_comment_reader.py +++ b/tests/unit/test_comment_reader.py @@ -134,12 +134,47 @@ def test_fetch_head_takes_the_newest_of_several_comments(): @pytest.mark.unit -def test_fetch_head_raises_on_an_empty_listing(): +def test_fetch_head_returns_none_on_an_empty_listing(): instance = MagicMock() instance.subreddit.return_value.comments.return_value = iter([]) with patch('racb.comment_reader.reddit.get_reddit_instance', return_value=instance): - with pytest.raises(RuntimeError): - comment_reader.fetch_head() + assert comment_reader.fetch_head() is None + + +@pytest.mark.unit +def test_an_empty_head_listing_is_skipped_and_the_next_poll_catches_up(): + reader, _ = make_reader(head=1000) + reader.poll_head() + head = {'value': None} + reader.fetch_head = lambda: head['value'] + reader.poll_head() + reader.poll_head() + assert reader.empty_head_polls == 2 + assert queued_blocks(reader) == [] + head['value'] = 1000 + 3 * BLOCK_SIZE + reader.poll_head() + assert reader.empty_head_polls == 0 + assert queued_blocks(reader) == [1001, 1101, 1201] + + +@pytest.mark.unit +def test_run_survives_an_empty_head_listing(): + errors = [] + polls = [] + + def empty_then_stop(): + polls.append(1) + if len(polls) == 2: + shutdown.request_stop() + return None + + reader, _ = make_reader() + reader.handle_exception = lambda error: errors.append(error) or True + reader.fetch_head = empty_then_stop + with patch('racb.comment_reader.HEAD_POLL_SECONDS', 0): + reader.run() + assert polls == [1, 1] + assert errors == [] @pytest.mark.unit