Honor a legacy CHARSET property parameter when parsing bytes

news/2078.bugfix

@@ -0,0 +1 @@
+Honor a legacy ``CHARSET`` property parameter when parsing bytes: a value that declares its encoding (e.g. ``SUMMARY;CHARSET=ISO-8859-1:...``) now decodes with that charset instead of coming back as U+FFFD replacement characters. I used AI to help with this change. @rahul05ranjan

src/icalendar/parser/content_line.py

@@ -18,6 +18,11 @@
 # to the first line break of a run, making a failed attempt O(1) instead of a
 # full rescan, while matching exactly the same strings.
 UFOLD = re.compile(r"(?:(?<!\n)\r\n|(?<![\r\n])\n)(?:\r?\n)*[ \t]")
+NEWLINE_BYTES = re.compile(rb"\r?\n")
+# a legacy CHARSET parameter declares the encoding of one property value
+CHARSET_PARAM_BYTES = re.compile(
+    rb";[ \t]*CHARSET[ \t]*=[ \t]*([^;:\r\n]+)", re.IGNORECASE
+)
 NEWLINE = re.compile(r"\r?\n")
 
 OWS = " \t"
@@ -121,6 +126,32 @@ def flush_pending() -> None:
     return "".join(out).strip()
 
 
+def _decode_with_charset(value: bytes) -> str:
+    """Decode content lines, honoring a legacy ``CHARSET`` parameter.
+
+    When the input is not valid UTF-8 as a whole, decode each content
+    line with the charset its ``CHARSET`` parameter declares, so a value
+    that names its encoding does not come back as U+FFFD. Lines without
+    a ``CHARSET`` parameter (or with an unknown one) keep the previous
+    best-effort UTF-8 decoding.
+    """
+    lines = []
+    for line in NEWLINE_BYTES.split(value):
+        # the parameters end at the first colon: a parameter VALUE quoting
+        # the text "CHARSET=..." must not select an encoding
+        head, _sep, _tail = line.partition(b":")
+        match = CHARSET_PARAM_BYTES.search(head)
+        if match:
+            charset = match.group(1).decode("ascii", "replace").strip().strip('"')
+            try:
+                lines.append(line.decode(charset))
+                continue
+            except (LookupError, UnicodeDecodeError):
+                pass
+        lines.append(to_unicode(line))
+    return "\r\n".join(lines)
+
+
 class Contentline(str):
     r"""A content line is a string that can be folded and parsed into parts.
 
@@ -318,7 +349,9 @@ def value_separator_index(self) -> int:
     @classmethod
     def from_ical(cls, ical, strict=False):
         """Unfold the content lines in an iCalendar into long content lines."""
-        ical = to_unicode(ical)
+        ical = (
+            _decode_with_charset(ical) if isinstance(ical, bytes) else to_unicode(ical)
+        )
         # a fold is carriage return followed by either a space or a tab
         return cls(UFOLD.sub("", ical), strict=strict)
 
@@ -342,7 +375,7 @@ def to_ical(self):
     @classmethod
     def from_ical(cls, st):
         """Parses a string into content lines."""
-        st = to_unicode(st)
+        st = _decode_with_charset(st) if isinstance(st, bytes) else to_unicode(st)
         try:
             # a fold is carriage return followed by either a space or a tab
             unfolded = UFOLD.sub("", st)

src/icalendar/tests/test_encoding.py

@@ -2,6 +2,8 @@
 
 import pytest
 
+from icalendar import Calendar
+
 
 @pytest.mark.parametrize(
     ("field", "expected_value"),
@@ -96,3 +98,46 @@ def test_apple_xlocation(calendars):
     """
     for event in calendars.x_location.walk("vevent"):
         assert len(event.errors) == 0, "Got too many errors"
+
+
+def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078():
+    """A legacy CHARSET parameter declares the encoding of the property
+    value: latin-1 bytes decode with the declared charset instead of
+    coming back as U+FFFD replacement characters (issue #2078).
+    """
+    ics = (
+        b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
+        b"BEGIN:VEVENT\r\nUID:1\r\nDTSTART:20261108T190000Z\r\n"
+        b"SUMMARY;CHARSET=ISO-8859-1:Caf\xe9 Meeting\r\n"
+        b"END:VEVENT\r\nEND:VCALENDAR\r\n"
+    )
+    event = Calendar.from_ical(ics).walk("VEVENT")[0]
+    assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting"
+
+
+def test_unknown_charset_falls_back_gracefully_issue_2078():
+    """An unknown CHARSET must not raise; the value degrades exactly as
+    it did before the CHARSET parameter was honored (issue #2078)."""
+    ics = (
+        b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
+        b"BEGIN:VEVENT\r\nUID:1\r\n"
+        b"SUMMARY;CHARSET=BOGUS-XYZ:Caf\xe9 Meeting\r\n"
+        b"END:VEVENT\r\nEND:VCALENDAR\r\n"
+    )
+    event = Calendar.from_ical(ics).walk("VEVENT")[0]
+    assert "Meeting" in str(event["SUMMARY"])
+
+
+def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078():
+    """UTF-8 text with a quoted parameter value mentioning CHARSET= must
+    still decode as UTF-8, not as the encoding named in the quotes."""
+    summary = "Caf\u00e9 Meeting"
+    ics = (
+        b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n"
+        b"BEGIN:VEVENT\r\nUID:1\r\n"
+        b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":'
+        + summary.encode("utf-8")
+        + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n"
+    )
+    event = Calendar.from_ical(ics).walk("VEVENT")[0]
+    assert str(event["SUMMARY"]) == summary