From a8eb8ab7fd64db85d0c48eae5ee2801131713717 Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Sun, 8 Nov 2026 21:57:27 +0000 Subject: [PATCH 1/2] fix(parser): unfold folded content lines before decoding bytes A content line folded in the middle of a multi-byte UTF-8 sequence was decoded with errors=replace before unfolding, so the split bytes turned into U+FFFD and the original value was unrecoverable. Unfold the raw bytes first (a byte-level twin of UFOLD) in Contentline.from_ical and Contentlines.from_ical, then decode the assembled line. String inputs take the previous path unchanged. Fixes #2077 AI: GLM-5.3 (z-ai, via OpenRouter). How used: AI coding assistant. I supplied the repro from issue #2077 and the fix design (unfold before decode, mirroring UFOLD at byte level); the model drafted the patch, the regression test and this message from my instructions. I reviewed the change and verified it by running the test suite. Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- news/2077.bugfix | 1 + src/icalendar/parser/content_line.py | 14 ++++++++++++++ src/icalendar/tests/test_encoding.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+) create mode 100644 news/2077.bugfix diff --git a/news/2077.bugfix b/news/2077.bugfix new file mode 100644 index 000000000..0075f01e9 --- /dev/null +++ b/news/2077.bugfix @@ -0,0 +1 @@ +Fixed a bug where a content line folded in the middle of a multi-byte UTF-8 character parsed to U+FFFD replacement characters: raw bytes are now unfolded before decoding, so such values round-trip losslessly (RFC 5545 section 3.1 anticipates such folds). I used AI to help with this change. @rahul05ranjan diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 00a7d5313..e7fd5356e 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -18,6 +18,12 @@ # to the first line break of a run, making a failed attempt O(1) instead of a # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(? int: @classmethod def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" + if isinstance(ical, bytes): + # unfold before decoding, so a fold inside a multi-byte UTF-8 + # character can be rejoined instead of being replaced + ical = UFOLD_BYTES.sub(b"", ical) ical = to_unicode(ical) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -311,6 +321,10 @@ def to_ical(self): @classmethod def from_ical(cls, st): """Parses a string into content lines.""" + if isinstance(st, bytes): + # unfold before decoding, so a fold inside a multi-byte UTF-8 + # character can be rejoined instead of being replaced + st = UFOLD_BYTES.sub(b"", st) st = to_unicode(st) try: # a fold is carriage return followed by either a space or a tab diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 4cc1557b0..1b1e49bab 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -2,6 +2,8 @@ import pytest +from icalendar import Calendar + @pytest.mark.parametrize( ("field", "expected_value"), @@ -96,3 +98,20 @@ def test_apple_xlocation(calendars): """ for event in calendars.x_location.walk("vevent"): assert len(event.errors) == 0, "Got too many errors" + + +def test_fold_inside_multibyte_utf8_sequence_issue_2077(): + """A fold in the middle of a multi-byte UTF-8 sequence must not + garble the value into U+FFFD replacement characters: the raw bytes + are unfolded before decoding (issue #2077). + """ + summary = "A" * 70 + "\U0001f389" + "B" * 20 + raw = ("SUMMARY:" + summary).encode("utf-8") + split = raw.index(b"\xf0\x9f") + 1 # inside the 4-byte sequence + folded = raw[:split] + b"\r\n " + raw[split:] + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + folded + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary From 34dcbd9726af7ed6062a655924c35a276b0e7ad9 Mon Sep 17 00:00:00 2001 From: Rahul Ranjan Date: Wed, 11 Nov 2026 05:16:48 +0000 Subject: [PATCH 2/2] REF: skip redundant string unfold for bytes input The string unfold after UFOLD_BYTES cannot match anything new: folds cannot reappear during decoding, so the second pass is a no-op for bytes input (verified with a 20000-case fuzz harness). Skip it and keep the single string unfold for str input, addressing the review comment. Co-authored-by: rahul05ranjan <11793455+rahul05ranjan@users.noreply.github.com> --- src/icalendar/parser/content_line.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index e7fd5356e..32c6a9350 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -295,8 +295,10 @@ def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" if isinstance(ical, bytes): # unfold before decoding, so a fold inside a multi-byte UTF-8 - # character can be rejoined instead of being replaced - ical = UFOLD_BYTES.sub(b"", ical) + # character can be rejoined instead of being replaced; folds + # cannot reappear during decoding, so the str unfold below is + # unnecessary for bytes input + return cls(to_unicode(UFOLD_BYTES.sub(b"", ical)), strict=strict) ical = to_unicode(ical) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -323,12 +325,16 @@ def from_ical(cls, st): """Parses a string into content lines.""" if isinstance(st, bytes): # unfold before decoding, so a fold inside a multi-byte UTF-8 - # character can be rejoined instead of being replaced - st = UFOLD_BYTES.sub(b"", st) - st = to_unicode(st) - try: + # character can be rejoined instead of being replaced; folds + # cannot reappear during decoding, so the str unfold below is + # unnecessary for bytes input + st = to_unicode(UFOLD_BYTES.sub(b"", st)) + unfolded = st + else: + st = to_unicode(st) # a fold is carriage return followed by either a space or a tab unfolded = UFOLD.sub("", st) + try: lines = cls(Contentline(line) for line in NEWLINE.split(unfolded) if line) lines.append("") # '\r\n' at the end of every content line except Exception as e: