diff --git a/news/2078.bugfix b/news/2078.bugfix new file mode 100644 index 000000000..c45ccfceb --- /dev/null +++ b/news/2078.bugfix @@ -0,0 +1 @@ +Honor a legacy ``CHARSET`` property parameter when parsing bytes: a value that declares its encoding (e.g. ``SUMMARY;CHARSET=ISO-8859-1:...``) now decodes with that charset instead of coming back as U+FFFD replacement characters. I used AI to help with this change. @rahul05ranjan diff --git a/src/icalendar/parser/content_line.py b/src/icalendar/parser/content_line.py index 9047546b0..b8942ba80 100644 --- a/src/icalendar/parser/content_line.py +++ b/src/icalendar/parser/content_line.py @@ -18,6 +18,11 @@ # to the first line break of a run, making a failed attempt O(1) instead of a # full rescan, while matching exactly the same strings. UFOLD = re.compile(r"(?:(? None: return "".join(out).strip() +def _decode_with_charset(value: bytes) -> str: + """Decode content lines, honoring a legacy ``CHARSET`` parameter. + + When the input is not valid UTF-8 as a whole, decode each content + line with the charset its ``CHARSET`` parameter declares, so a value + that names its encoding does not come back as U+FFFD. Lines without + a ``CHARSET`` parameter (or with an unknown one) keep the previous + best-effort UTF-8 decoding. + """ + lines = [] + for line in NEWLINE_BYTES.split(value): + # the parameters end at the first colon: a parameter VALUE quoting + # the text "CHARSET=..." must not select an encoding + head, _sep, _tail = line.partition(b":") + match = CHARSET_PARAM_BYTES.search(head) + if match: + charset = match.group(1).decode("ascii", "replace").strip().strip('"') + try: + lines.append(line.decode(charset)) + continue + except (LookupError, UnicodeDecodeError): + pass + lines.append(to_unicode(line)) + return "\r\n".join(lines) + + class Contentline(str): r"""A content line is a string that can be folded and parsed into parts. @@ -318,7 +349,9 @@ def value_separator_index(self) -> int: @classmethod def from_ical(cls, ical, strict=False): """Unfold the content lines in an iCalendar into long content lines.""" - ical = to_unicode(ical) + ical = ( + _decode_with_charset(ical) if isinstance(ical, bytes) else to_unicode(ical) + ) # a fold is carriage return followed by either a space or a tab return cls(UFOLD.sub("", ical), strict=strict) @@ -342,7 +375,7 @@ def to_ical(self): @classmethod def from_ical(cls, st): """Parses a string into content lines.""" - st = to_unicode(st) + st = _decode_with_charset(st) if isinstance(st, bytes) else to_unicode(st) try: # a fold is carriage return followed by either a space or a tab unfolded = UFOLD.sub("", st) diff --git a/src/icalendar/tests/test_encoding.py b/src/icalendar/tests/test_encoding.py index 4cc1557b0..993bc002f 100644 --- a/src/icalendar/tests/test_encoding.py +++ b/src/icalendar/tests/test_encoding.py @@ -2,6 +2,8 @@ import pytest +from icalendar import Calendar + @pytest.mark.parametrize( ("field", "expected_value"), @@ -96,3 +98,46 @@ def test_apple_xlocation(calendars): """ for event in calendars.x_location.walk("vevent"): assert len(event.errors) == 0, "Got too many errors" + + +def test_charset_parameter_decodes_legacy_non_utf8_value_issue_2078(): + """A legacy CHARSET parameter declares the encoding of the property + value: latin-1 bytes decode with the declared charset instead of + coming back as U+FFFD replacement characters (issue #2078). + """ + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\nDTSTART:20261108T190000Z\r\n" + b"SUMMARY;CHARSET=ISO-8859-1:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == "Caf\u00e9 Meeting" + + +def test_unknown_charset_falls_back_gracefully_issue_2078(): + """An unknown CHARSET must not raise; the value degrades exactly as + it did before the CHARSET parameter was honored (issue #2078).""" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b"SUMMARY;CHARSET=BOGUS-XYZ:Caf\xe9 Meeting\r\n" + b"END:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert "Meeting" in str(event["SUMMARY"]) + + +def test_charset_text_inside_quoted_param_value_is_not_a_directive_issue_2078(): + """UTF-8 text with a quoted parameter value mentioning CHARSET= must + still decode as UTF-8, not as the encoding named in the quotes.""" + summary = "Caf\u00e9 Meeting" + ics = ( + b"BEGIN:VCALENDAR\r\nVERSION:2.0\r\nPRODID:-//t//EN\r\n" + b"BEGIN:VEVENT\r\nUID:1\r\n" + b'SUMMARY;ALTREP="cid:x;CHARSET=ISO-8859-1":' + + summary.encode("utf-8") + + b"\r\nEND:VEVENT\r\nEND:VCALENDAR\r\n" + ) + event = Calendar.from_ical(ics).walk("VEVENT")[0] + assert str(event["SUMMARY"]) == summary