Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 73 additions & 1 deletion src/memos/mem_scheduler/task_schedule_modules/redis_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,69 @@ def _parse_pending_entry(self, entry) -> tuple[str, int]:
# Assume object (redis-py 5.x+ PendingMessage)
return getattr(entry, "message_id", None), getattr(entry, "time_since_delivered", 0)

def _parse_pending_deliveries(self, entry) -> int:
"""Extract times_delivered from a pending entry."""
if isinstance(entry, dict):
return entry.get("times_delivered", 0) or 0
elif isinstance(entry, tuple | list):
return entry[3] if len(entry) > 3 else 0
return getattr(entry, "times_delivered", 0) or 0

def _filter_poison_messages(
self, stream_key: str, claimed: list[tuple[str, dict]]
) -> list[tuple[str, dict]]:
"""2026-09-14 毒消息隔离(用户批准):交付次数 > SCHEDULER_MAX_DELIVERIES
(默认5)的消息 = 永久失败循环(实证:09-14 daike 批次输入超长 400/20015
每轮必败、无限重排,烧判别配额+内存翻滚)。隔离动作:写入死信流
scheduler:messages:stream:deadletter(保留证据)+ xack + xdel,
不再投回处理器。查询失败时放行(宁多跑一次不错杀)。"""
if not claimed or not self._redis_conn:
return claimed
max_deliveries = int(os.getenv("SCHEDULER_MAX_DELIVERIES", "5"))
try:
pend = {}
for e in self._redis_conn.xpending_range(
stream_key, self.consumer_group, "-", "+", 1000
):
mid, _ = self._parse_pending_entry(e)
if mid:
pend[mid] = self._parse_pending_deliveries(e)
except Exception as e:
logger.debug(f"[PoisonGuard] xpending query failed on {stream_key}: {e}")
return claimed
good, dropped = [], []
for mid, fields in claimed:
deliveries = pend.get(mid, 0)
if deliveries > max_deliveries:
dropped.append((mid, fields, deliveries))
else:
good.append((mid, fields))
for mid, fields, deliveries in dropped:
try:
self._redis_conn.xadd(
"scheduler:messages:stream:deadletter",
{
"src_stream": stream_key,
"orig_id": str(mid),
"deliveries": str(deliveries),
"payload": str(fields)[:4000],
"dropped_at": str(time.time()),
},
maxlen=500,
approximate=True,
)
self._redis_conn.xack(stream_key, self.consumer_group, mid)
self._redis_conn.xdel(stream_key, mid)
except Exception as e:
logger.warning(f"[PoisonGuard] drop failed for {mid}: {e}")
if dropped:
logger.warning(
f"[PoisonGuard] {stream_key}: quarantined {len(dropped)} poisoned message(s) "
f"(deliveries>{max_deliveries}) -> deadletter stream"
)
return good


def _manual_xautoclaim(
self, stream_key: str, min_idle_time: int, count: int
) -> tuple[str, list[tuple[str, dict]], list[str]]:
Expand Down Expand Up @@ -844,6 +907,9 @@ def _claim_pending_messages(
f"Unexpected xautoclaim response length: {len(claimed_result)}"
)

# Quarantine poison messages (deliveries > limit) to a
# dead-letter stream instead of infinite redelivery.
claimed = self._filter_poison_messages(stream_key, claimed)
return [(stream_key, claimed)] if claimed else []
except Exception as read_err:
err_msg = str(read_err).lower()
Expand Down Expand Up @@ -871,7 +937,10 @@ def _claim_pending_messages(
f"Unexpected xautoclaim response length: {len(claimed_result)}"
) from read_err

return [(stream_key, claimed)] if claimed else []
# Quarantine poison messages (deliveries > limit) to a
# dead-letter stream instead of infinite redelivery.
claimed = self._filter_poison_messages(stream_key, claimed)
return [(stream_key, claimed)] if claimed else []
return []

# Fallback to manual xautoclaim for older Redis versions
Expand All @@ -891,6 +960,9 @@ def _claim_pending_messages(
_next, claimed, _deleted = self._manual_xautoclaim(
stream_key, min_idle, need_pending_count
)
# Quarantine poison messages (deliveries > limit) to a
# dead-letter stream instead of infinite redelivery.
claimed = self._filter_poison_messages(stream_key, claimed)
return [(stream_key, claimed)] if claimed else []
except Exception:
return []
Expand Down