Unfold folded content lines before decoding bytes

news/2077.bugfix

@@ -0,0 +1 @@
+Fixed a bug where a content line folded in the middle of a multi-byte UTF-8 character parsed to U+FFFD replacement characters: raw bytes are now unfolded before decoding, so such values round-trip losslessly (RFC 5545 section 3.1 anticipates such folds). I used AI to help with this change. @rahul05ranjan

src/icalendar/parser/content_line.py

@@ -18,6 +18,12 @@
 # to the first line break of a run, making a failed attempt O(1) instead of a
 # full rescan, while matching exactly the same strings.
 UFOLD = re.compile(r"(?:(?<!\n)\r\n|(?<![\r\n])\n)(?:\r?\n)*[ \t]")
+# The same pattern on raw bytes, to unfold an undecodable input before
+# decoding it: a producer may fold a line in the middle of a multi-byte
+# UTF-8 sequence, and decoding first would replace those bytes with
+# U+FFFD before they can be rejoined (RFC 5545 section 3.1 anticipates
+# that producers may split multi-octet characters).
+UFOLD_BYTES = re.compile(rb"(?:(?<!\n)\r\n|(?<![\r\n])\n)(?:\r?\n)*[ \t]")
 NEWLINE = re.compile(r"\r?\n")
 
 OWS = " \t"
@@ -318,6 +324,12 @@ def value_separator_index(self) -> int:
     @classmethod
     def from_ical(cls, ical, strict=False):
         """Unfold the content lines in an iCalendar into long content lines."""
+        if isinstance(ical, bytes):
+            # unfold before decoding, so a fold inside a multi-byte UTF-8
+            # character can be rejoined instead of being replaced; folds
+            # cannot reappear during decoding, so the str unfold below is
+            # unnecessary for bytes input
+            return cls(to_unicode(UFOLD_BYTES.sub(b"", ical)), strict=strict)
         ical = to_unicode(ical)
         # a fold is carriage return followed by either a space or a tab
         return cls(UFOLD.sub("", ical), strict=strict)
@@ -342,10 +354,18 @@ def to_ical(self):
     @classmethod
     def from_ical(cls, st):
         """Parses a string into content lines."""
-        st = to_unicode(st)
-        try:
+        if isinstance(st, bytes):
+            # unfold before decoding, so a fold inside a multi-byte UTF-8
+            # character can be rejoined instead of being replaced; folds
+            # cannot reappear during decoding, so the str unfold below is
+            # unnecessary for bytes input
+            st = to_unicode(UFOLD_BYTES.sub(b"", st))
+            unfolded = st
+        else:
+            st = to_unicode(st)
             # a fold is carriage return followed by either a space or a tab
             unfolded = UFOLD.sub("", st)
+        try:
             lines = cls(Contentline(line) for line in NEWLINE.split(unfolded) if line)
             lines.append("")  # '\r\n' at the end of every content line
         except Exception as e:

src/icalendar/tests/test_encoding.py

@@ -2,6 +2,8 @@
 
 import pytest
 
+from icalendar import Calendar
+
 
 @pytest.mark.parametrize(
     ("field", "expected_value"),
@@ -96,3 +98,20 @@ def test_apple_xlocation(calendars):
     """
     for event in calendars.x_location.walk("vevent"):
         assert len(event.errors) == 0, "Got too many errors"
+
+
+def test_fold_inside_multibyte_utf8_sequence_issue_2077():
+    """A fold in the middle of a multi-byte UTF-8 sequence must not
+    garble the value into U+FFFD replacement characters: the raw bytes
+    are unfolded before decoding (issue #2077).
+    """
+    summary = "A" * 70 + "\U0001f389" + "B" * 20
+    raw = ("SUMMARY:" + summary).encode("utf-8")
+    split = raw.index(b"\xf0\x9f") + 1  # inside the 4-byte sequence
+    folded = raw[:split] + b"\r\n " + raw[split:]
+    ics = (
+        b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
+        b"BEGIN:VEVENT\r\nUID:1\r\n" + folded + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"
+    )
+    event = Calendar.from_ical(ics).walk("VEVENT")[0]
+    assert str(event["SUMMARY"]) == summary