diff --git a/news/2077.bugfix b/news/2077.bugfix new file mode 100644 index 000000000..0075f01e9 --- /dev/null +++ b/news/2077.bugfix @@ -0,0 +1 @@ +Fixed a bug where a content line folded in the middle of a multi-byte UTF-8 character parsed to U+FFFD replacement characters: raw bytes are now unfolded before decoding, so such values round-trip losslessly (RFC 5545 section 3.1 anticipates such folds). I used AI to help with this change. @rahul05ranjan diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 9047546b0..203281e25 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -18,6 +18,12 @@ # to the first line break of a run, making a failed attempt O(1) instead of a # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(? int: @classmethod def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" + if isinstance(ical, bytes): + # unfold before decoding, so a fold inside a multi-byte UTF-8 + # character can be rejoined instead of being replaced; folds + # cannot reappear during decoding, so the str unfold below is + # unnecessary for bytes input + return cls(to_unicode(UFOLD_BYTES.sub(b"", ical)), strict=strict) ical = to_unicode(ical) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -342,10 +354,18 @@ def to_ical(self): @classmethod def from_ical(cls, st): """Parses a string into content lines.""" - st = to_unicode(st) - try: + if isinstance(st, bytes): + # unfold before decoding, so a fold inside a multi-byte UTF-8 + # character can be rejoined instead of being replaced; folds + # cannot reappear during decoding, so the str unfold below is + # unnecessary for bytes input + st = to_unicode(UFOLD_BYTES.sub(b"", st)) + unfolded = st + else: + st = to_unicode(st) # a fold is carriage return followed by either a space or a tab unfolded = UFOLD.sub("", st) + try: lines = cls(Contentline(line) for line in NEWLINE.split(unfolded) if line) lines.append("") # '\r\n' at the end of every content line except Exception as e: diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 49c14d46d..a74c27a5c 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -2,6 +2,8 @@ import pytest +from icalendar import Calendar + @pytest.mark.parametrize( ("field", "expected_value"), @@ -96,3 +98,20 @@ def test_apple_xlocation(calendars): """ for event in calendars.x_location.walk("vevent"): assert len(event.errors) == 0, "Got too many errors" + + +def test_fold_inside_multibyte_utf8_sequence_issue_2077(): + """A fold in the middle of a multi-byte UTF-8 sequence must not + garble the value into U+FFFD replacement characters: the raw bytes + are unfolded before decoding (issue #2077). + """ + summary = "A" * 70 + "\U0001f389" + "B" * 20 + raw = ("SUMMARY:" + summary).encode("utf-8") + split = raw.index(b"\xf0\x9f") + 1 # inside the 4-byte sequence + folded = raw[:split] + b"\r\n " + raw[split:] + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + folded + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary