diff --git a/docs/design/API_DESIGN.md b/docs/design/API_DESIGN.md index 2de8487..e5cb950 100644 --- a/docs/design/API_DESIGN.md +++ b/docs/design/API_DESIGN.md @@ -1218,7 +1218,11 @@ from both is dropped) are enough for the generic layer to report the losses itself, so a backend whose mapping is that simple writes no mapper. A `LOSSY` entry without `values` is allowed, for a backend with mapping rules of its own; the conformance suite can then check only that each value is -either stored or refused before any I/O. +either stored or refused before any I/O. A property kept only beside +another is `LOSSY` with `requires`, the properties it needs: Zimbra keeps a +task's `DTSTART` only where it has a `DUE`, so `task.start` there is +`{"level": "lossy", "requires": ["DUE"]}`, and the generic layer reports a +`DTSTART` without a `DUE` as lost. `supports()` counts `UNKNOWN` as not supported: it is the view for a caller deciding whether to rely on a feature, and nothing promises an unprobed one @@ -1236,6 +1240,7 @@ initial set: | `collection.set-name` | `set_name()` | | `collection.mixed` | one collection holds several component kinds; `UNSUPPORTED` where each holds one (CCS, a task tracker), and `create_collection()` then makes one for the first kind asked, in sorted order, of those the backend holds | | `search.server-side`, `search.time-range`, `search.text` | how much the server filters; never affects results ([§4 Search](#4-search)) | +| `listing.complete` | a listing (`search()`, `iter_search()`, the listing shorthands, `changes()`) returns every stored object, however many; `UNSUPPORTED` where the server cuts a long one at a limit of its own: the listing then raises `ListingLimitError` where the server says so with a 507 (Stalwart, at its `WebDav.maxResults`, 2000 by default), and is short where it does not | | `changes` | `changes()`: `FULL` with a native token, `EMULATED` by listing | | `changes.known` | `changes(known=...)` fetches only what changed: `FULL` where etags can be listed without content (CalDAV, vdir), `EMULATED` where everything is read and compared (JMAP, single `.ics` file, feed, Gitea) | | `write.wire` | `add(wire=...)`, `save(wire=...)`: given bytes are sent as they are; `FULL` on vdir, and on CalDAV through caldav's `save(raw_data=...)` (caldav 3.6, [caldav #763](https://github.com/python-caldav/caldav/issues/763)); `UNSUPPORTED` elsewhere | @@ -1244,6 +1249,7 @@ initial set: | `identity.client-native-id` | `add(native_id=...)`: the caller may name the stored object (CalDAV href, vdir file name) | | `properties.passthrough` | unknown properties and components survive a round trip | | `recurrence`, `recurrence.edit-this-and-future` | stores a recurrence set: `RRULE`, `RDATE`, `EXDATE` and the overrides (`RECURRENCE-ID`); `UNSUPPORTED` drops all four, overrides included; [§3.7 Recurrence](#37-recurrence) | +| `recurrence.rdate` | an event's `RDATE` survives where `recurrence` stores one; `UNSUPPORTED` drops it, and the overrides of the occurrences only it gives, the `RRULE` kept (Zimbra) | | `recurrence.edit-occurrence` | `save(occurrence)` merges it into the stored object ([§3.7](#37-recurrence)); `UNSUPPORTED` where the merge is not available yet (Q3) | | `task.complete-recurring` | `complete()` on a recurring task or one occurrence of it ([§3.7](#37-recurrence)) | | `task.uncomplete` | `uncomplete()` makes a completed task pending again; `UNSUPPORTED` where the server keeps it completed (GMX) | @@ -1252,8 +1258,9 @@ initial set: | `task.start`, `task.due`, `task.completed`, `task.planned`, `task.estimate` | | | `task.relations.parent`, `task.relations.depends-on` | | | `categories`, `attendees` | for every component; on a task, attendees are its assignees | -| `alarms` | `VALARM` components survive; `UNSUPPORTED` drops them | +| `alarms` | `VALARM` components survive; `UNSUPPORTED` drops them; `LOSSY` sends them as they are, to come back rewritten, which `verify` lets pass (Ox makes a task's alarm trigger absolute and replaces its description) | | `timezones` | a time with a `TZID` keeps it (with its `VTIMEZONE`), a floating time stays floating; `LOSSY` stores both in UTC, a floating time read as local time ([§4](#4-search)) | +| `timezones.floating` | a floating time stays floating, whatever `timezones` says of a time with a `TZID`; `LOSSY` stores it in UTC, read as local time (Zimbra and SOGo keep a `TZID` but not a floating time) | | `dates` | a `DATE` value stays a date; `LOSSY` stores midnight UTC | | `time-log` | the time log of a task or event ("Time tracking" in [§3.4](#34-task)); `LOSSY` with `details["lost"]`, or `details["total-only"]` | @@ -1326,6 +1333,8 @@ Three cases, matching the roadmap's "raise, degrade, or emulate": property: str # e.g. "PRIORITY", "STATUS", "RELATED-TO" sent: object # the value in the item stored: object | None # what the backend will store instead; None if dropped + note: str | None = None # how the server changes it, where the backend cannot say + # what it stores (Ox rewrites a task's alarm): stored is sent class LossPolicy(Protocol): def __call__(self, losses: Sequence[Loss], item: Item) -> None: ... @@ -1359,8 +1368,10 @@ property that was sent, after normalisation, except those the mapper had already reported as not storable (a lossy write, above), and skips what servers legitimately change: `DTSTAMP`, `SEQUENCE`, `LAST-MODIFIED`, property and parameter order, a parameter spelt out at its RFC 5545 default -(`PARTSTAT=NEEDS-ACTION`, `RSVP=FALSE`, ...), and the identity a backend -assigns (`uid`, `native_id`, `etag`; +(`PARTSTAT=NEEDS-ACTION`, `RSVP=FALSE`, ...), a parameter the server +adds (SOGo's `RSVP=TRUE`, Ox's `EMAIL=`: that loses nothing), an +attendee it adds (Ox adds the organizer, as a scheduling server does), +and the identity a backend assigns (`uid`, `native_id`, `etag`; [§3.2 Identity](#32-identity)). A difference raises `VerificationError` listing what the server dropped or changed. It is not a `LossyWriteError`: that one means nothing was sent, while a `VerificationError` means the @@ -1389,7 +1400,8 @@ CalendaringError ├── RateLimitError # carries .retry_after: float | None ├── VerificationError # verify=True: written, but stored differently; carries .differences └── BackendError # anything else from the backend or transport - └── TransportError # network, TLS, timeout + ├── TransportError # network, TLS, timeout + └── ListingLimitError # the server cut a listing at a limit of its own; nothing of it is returned CalendaringWarning └── LossyWriteWarning @@ -1434,6 +1446,13 @@ CalendaringWarning it; a caller that can live with the gap takes `.results`. The listing primitives return a `BrokenObject` in place of a `StoredObject` rather than raising, so the backend decides per object. +- **A cut listing is not a partial result.** `ListingLimitError` is a + `BackendError`, not a `PartialResultError`: the server answered a 507 for + the collection, and caldav keeps none of what came before it, so there + is no `.results` to hand on and no `.broken` to name. Where the + backend declares `listing.complete` unsupported, a caller expecting + more objects than the server's limit narrows the search itself, e.g. by + time range. - Fan-out ([§2.1 Workspace](#21-workspace)) does not raise; `MultiResult.raise_for_errors()` raises an `ExceptionGroup` of these. `Workspace.search()` keeps a `PartialResultError`'s `.results` among the items, and the error among the errors. caldav's errors map one to one where they overlap: @@ -1442,8 +1461,9 @@ caldav's errors map one to one where they overlap: `DAVError` carrying the `DAV:valid-sync-token` precondition, any 412, and a 500 from a server flagged as above → `SyncTokenError`; `AuthorizationError` → `AuthenticationError` or `AuthorizationError` by -status code, `RateLimitError` → `RateLimitError`, other `DAVError` → -`BackendError`. +status code, `RateLimitError` → `RateLimitError`, a 507 in a listing (inside +the multistatus, or for the whole `REPORT` or `PROPFIND`) → `ListingLimitError`, +other `DAVError` → `BackendError`. --- diff --git a/news/+added-parameters.bugfix.rst b/news/+added-parameters.bugfix.rst new file mode 100644 index 0000000..64e9ba6 --- /dev/null +++ b/news/+added-parameters.bugfix.rst @@ -0,0 +1 @@ +``verify=True`` and the conformance suite no longer count a parameter the server adds as a difference where it says nothing about the value: SOGo adding ``RSVP=TRUE`` to an ATTENDEE, or Ox adding ``EMAIL=``, loses nothing, and neither does Ox adding the organizer as an attendee. A parameter the server drops or changes still is one, and so is one it adds with a meaning of its own, such as a TZID or ``PARTSTAT=DECLINED``. diff --git a/news/+completed-requires-status.bugfix.rst b/news/+completed-requires-status.bugfix.rst new file mode 100644 index 0000000..1aef957 --- /dev/null +++ b/news/+completed-requires-status.bugfix.rst @@ -0,0 +1 @@ +On a server that drops a task's completion time where the task has no status (Ox), ``task.completed`` is declared ``lossy``, and such a write is reported as losing the completion time instead of losing it silently. diff --git a/news/+dropped-start.bugfix.rst b/news/+dropped-start.bugfix.rst new file mode 100644 index 0000000..31e8961 --- /dev/null +++ b/news/+dropped-start.bugfix.rst @@ -0,0 +1 @@ +``verify=True`` reports a start, end or due time the server dropped as a ``VerificationError``, instead of failing with ``KeyError``. diff --git a/news/+dropped-vtimezone.bugfix.rst b/news/+dropped-vtimezone.bugfix.rst new file mode 100644 index 0000000..4167a20 --- /dev/null +++ b/news/+dropped-vtimezone.bugfix.rst @@ -0,0 +1 @@ +A CalDAV read puts back a VTIMEZONE the server dropped (CCS), from zoneinfo; ``wire`` keeps the bytes as received. diff --git a/news/+listing-limit.feature.rst b/news/+listing-limit.feature.rst new file mode 100644 index 0000000..67421f4 --- /dev/null +++ b/news/+listing-limit.feature.rst @@ -0,0 +1 @@ +New capability ``listing.complete`` and error ``ListingLimitError``: a server that cuts a long listing at a limit of its own (Stalwart, at its ``WebDav.maxResults``, 2000 objects by default) declares ``listing.complete`` unsupported, and a listing that hits the limit raises ``ListingLimitError`` instead of a ``BackendError`` where the server answers it with a 507. None of the listing is returned. diff --git a/news/+loss-note.feature.rst b/news/+loss-note.feature.rst new file mode 100644 index 0000000..e0d339f --- /dev/null +++ b/news/+loss-note.feature.rst @@ -0,0 +1 @@ +A Loss has a note saying how the server changes a value where the backend cannot tell what it will store; the warn policy shows it. diff --git a/news/+occurrence-restamped.bugfix.rst b/news/+occurrence-restamped.bugfix.rst new file mode 100644 index 0000000..d8eda3b --- /dev/null +++ b/news/+occurrence-restamped.bugfix.rst @@ -0,0 +1 @@ +Deleting an occurrence of a recurring event no longer fails with ``ConflictError`` on servers that re-stamp every occurrence when one is deleted (Ox). diff --git a/news/+recurrence-rdate.feature.rst b/news/+recurrence-rdate.feature.rst new file mode 100644 index 0000000..cc1556a --- /dev/null +++ b/news/+recurrence-rdate.feature.rst @@ -0,0 +1 @@ +New capability ``recurrence.rdate``: whether an ``RDATE`` survives where an ``RRULE`` does. The CalDAV backend declares it unsupported on Zimbra, which drops every ``RDATE``, so the loss policy hears of it instead of the date silently disappearing. diff --git a/news/+task-alarm-rewritten.bugfix.rst b/news/+task-alarm-rewritten.bugfix.rst new file mode 100644 index 0000000..ae4bbb0 --- /dev/null +++ b/news/+task-alarm-rewritten.bugfix.rst @@ -0,0 +1 @@ +On a server that rewrites a task's alarm (Ox makes its trigger absolute and replaces its description), ``alarms`` is declared ``lossy`` on a task list, and such a write is reported as changing the alarm instead of failing verification. diff --git a/news/+task-start-requires-due.bugfix.rst b/news/+task-start-requires-due.bugfix.rst new file mode 100644 index 0000000..1a0148b --- /dev/null +++ b/news/+task-start-requires-due.bugfix.rst @@ -0,0 +1 @@ +On a server that drops a task's start time where the task has no due time (Zimbra), ``task.start`` is declared ``lossy``, and a task with a start but no due is reported as losing its start instead of losing it silently. diff --git a/news/+timezones-floating.feature.rst b/news/+timezones-floating.feature.rst new file mode 100644 index 0000000..0700623 --- /dev/null +++ b/news/+timezones-floating.feature.rst @@ -0,0 +1 @@ +New capability ``timezones.floating``: whether a floating time stays floating. The CalDAV backend declares it ``lossy`` where the server stores a floating time in UTC but keeps a ``TZID`` (Zimbra, SOGo), so such a time is written in UTC, read as local time, instead of being reported as changed. diff --git a/src/calendaring/__init__.py b/src/calendaring/__init__.py index bc5b8d4..6008238 100644 --- a/src/calendaring/__init__.py +++ b/src/calendaring/__init__.py @@ -38,6 +38,7 @@ ConfigurationError, ConflictError, InvalidDataError, + ListingLimitError, LossyWriteError, LossyWriteWarning, NotFoundError, @@ -89,6 +90,7 @@ "Event", "Feature", "InvalidDataError", + "ListingLimitError", "Item", "Journal", "Loss", diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 950b6b3..8f1b32f 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -86,6 +86,8 @@ def _stored(self, obj: Any) -> StoredObject: # raw_data is what the server sent; data is what caldav made of it, which is what gets parsed. data = obj.data parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) + # CCS drops a VTIMEZONE, which RFC 5545 section 3.2.19 requires for each TZID; zoneinfo has it. + parsed.add_missing_timezones() return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) def _listed(self, obj: Any) -> StoredObject | BrokenObject: diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index 26f9c48..0b18a99 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -25,7 +25,7 @@ import recurring_ical_events from icalendar_searcher import Searcher -from .._mapping import Change, differences, plan_calendar_losses, plan_losses +from .._mapping import NOT_VERIFIED, Change, differences, plan_calendar_losses, plan_losses, undeclared from ..capabilities import BACKEND_FEATURES, COMPONENT_FEATURE, Capabilities, Feature, Support from ..errors import ( AlreadyExistsError, @@ -93,6 +93,14 @@ def _occurrence_now(calendar: icalendar.Calendar, item: Item) -> icalendar.Compo return None +def _unstamped(component: icalendar.Component) -> icalendar.Component: + """``component`` without what a server rewrites in every component of an object it stores (Ox).""" + out = copy.deepcopy(component) + for name in NOT_VERIFIED: + out.pop(name, None) + return out + + def _occurs_at(item: Item) -> datetime: """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" rid = item.component.get("RECURRENCE-ID") @@ -431,7 +439,7 @@ def _prepare( async def _check(self, sent: icalendar.Calendar, result: AsyncItem) -> None: assert result.native_id is not None stored = await self._fetch(result.native_id) - found = differences(sent, stored.calendar) + found = undeclared(differences(sent, stored.calendar), self.capabilities) if found: raise VerificationError(found) @@ -819,7 +827,8 @@ async def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: Siblings share the stored object's etag, so a stale etag is not a conflict by itself: only a change to this occurrence on the server is. It is found by comparing the stored occurrence with the item as the - caller holds it, so once the etag moved, a local edit before + caller holds it, leaving out what verify never compares (DTSTAMP, SEQUENCE, + LAST-MODIFIED, UID), so once the etag moved, a local edit before ``delete()`` also counts as one; with the etag unchanged nothing changed on the server, and a local edit is deleted with the rest. An occurrence already gone is ``NotFoundError``. @@ -830,7 +839,7 @@ async def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: now = _occurrence_now(stored.calendar, item) if now is None: raise NotFoundError(f"the occurrence {item.component.decoded('RECURRENCE-ID')} is gone") - if not overwrite and now != item.component: + if not overwrite and _unstamped(now) != _unstamped(item.component): raise ConflictError( f"the occurrence {item.component.decoded('RECURRENCE-ID')} changed since it was read" ) diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index f6ff83a..2ab0689 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -47,6 +47,7 @@ async def open_backend(self) -> AsyncBackend: BackendError, CalendaringError, ConflictError, + ListingLimitError, LossyWriteError, LossyWriteWarning, NotFoundError, @@ -773,8 +774,8 @@ async def test_property_round_trips_as_declared(self, collection: AsyncCollectio assert support is Support.LOSSY, f"{feature} is declared {support.value} but the write is lossy" return read = await collection.reload(stored) - if support is not Support.LOSSY: - assert property_lines(read.component, name) == property_lines(item.component, name) + if support is not Support.LOSSY: # stored as verify sees it: a parameter the server adds loses nothing + assert [d for d in differences(item.icalendar, read.icalendar) if d.property == name] == [] @pytest.mark.parametrize("feature", list(VALUE_SAMPLES)) async def test_values_map_as_declared(self, collection: AsyncCollection, feature: Feature) -> None: @@ -921,6 +922,14 @@ def fields(items: Sequence[AsyncItem]) -> list[tuple[Any, ...]]: async def test_listings_are_not_cut_at_a_page(self, collection: AsyncCollection) -> None: kind = next(k for k in KINDS if holds(collection, k)) seeded = await self.seed(collection, [minimal(kind, f"paged {n}") for n in range(self.page_size_hint)]) + try: + await self._listings_are_complete(collection, seeded) + except ListingLimitError: + # Declared: the server cuts a long listing, and says so (Stalwart, at its WebDav.maxResults) + if level(collection, Feature.LISTING_COMPLETE) is not Support.UNSUPPORTED: + raise + + async def _listings_are_complete(self, collection: AsyncCollection, seeded: list[AsyncItem]) -> None: mine = {nid(i) for i in seeded} assert {nid(i) for i in stored_by(await collection.search(), seeded)} == mine assert {nid(i) async for i in collection.iter_search() if i.native_id in mine} == mine diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index 4a47d05..ff334d0 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -51,9 +51,25 @@ Feature.TASK_RELATIONS_PARENT: "save-load.icalendar.related-to", Feature.TASK_RELATIONS_DEPENDS_ON: "save-load.icalendar.related-to", Feature.TIMEZONES: "save-load.event.timezone", + Feature.TIMEZONES_FLOATING: "save-load.event.floating", + Feature.RECURRENCE_RDATE: "save-load.event.recurrences.rdate", Feature.TASK_UNCOMPLETE: "save-load.todo.uncomplete", } +# Feature -> the caldav feature saying a task keeps the property only beside others, those, and a note. +REQUIRES_FROM_SERVER = { + Feature.TASK_START: ( + "save-load.todo.dtstart-without-due", + ["DUE"], + "the server drops the DTSTART of a task without a DUE", # Zimbra + ), + Feature.TASK_COMPLETED: ( + "save-load.todo.completed-without-status", + ["STATUS"], + "the server drops the COMPLETED of a task without a STATUS", # Ox + ), +} + # Feature -> the caldav feature probed on a task. A level is the collection's, not a component's, so # a calendar holding events too keeps these: lowering them would cost its events what they can keep. TASK_FROM_SERVER = { @@ -109,6 +125,10 @@ def _level(features: Any, name: str) -> Support: return LEVELS.get(str(features.is_supported(name, str)), X) +# Where the server may cut a listing short (Stalwart: WebDav.maxResults, with a 507). +LISTING_LIMITS = ("search.limit", "propfind.limit", "multiget.limit") + + # What the server must store, or honour, as sent: FULL on a hinted server, UNKNOWN on another. STORED = frozenset( { @@ -140,6 +160,16 @@ def _table(features: Any) -> dict[Feature, Any]: table.update(dict.fromkeys(STORED, X)) for feature, name in FROM_SERVER.items(): table[feature] = _level(features, name) + limits = {_level(features, name) for name in LISTING_LIMITS} + table[Feature.LISTING_COMPLETE] = ( + {"level": U, "note": "the server cuts a long listing at a limit of its own"} + if U in limits + else X + if X in limits + else F + ) + if table[Feature.TIMEZONES_FLOATING] is U: # stored, in UTC: Zimbra and SOGo + table[Feature.TIMEZONES_FLOATING] = {"level": L, "note": "a floating time is stored in UTC"} if table[Feature.CREATE_COLLECTION] is F: notes = [] if _level(features, "create-calendar.with-supported-component-types") is U: @@ -152,7 +182,8 @@ def _table(features: Any) -> dict[Feature, Any]: notes.append("made for one component only, the first in sorted order") if notes: table[Feature.CREATE_COLLECTION] = {"level": L, "note": "; ".join(notes)} - if _level(features, "save.if-none-match") is U: # the server overwrites a name already taken + # The server overwrites a name already taken, or refuses it otherwise than with 412 (Bedework: 500). + if _level(features, "save.if-none-match") is U: # caldav then GETs the name before the PUT where a duplicate UID is refused and names are kept (SOGo), # and otherwise searches by UID, which leaves the name unguarded if str(features.is_supported("save.no-uid-conflict", str)) in {"full", "quirk", "ungraceful"} and ( @@ -261,6 +292,18 @@ def backend_capabilities(features: Any) -> Capabilities: return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) +def _alarm(features: Any) -> Any: + """What a task's VALARM becomes: dropped, or rewritten (Ox: an absolute trigger and its own description).""" + level = _level(features, "save-load.todo.alarm") + if level is not U: + return None + hint = features.is_supported("save-load.todo.alarm", dict) + if hint.get("support") != "broken": + return U + note = "the server rewrites a task's alarm" + return {"level": L, "note": f"{note}: {hint['behaviour']}" if hint.get("behaviour") else note} + + def collection_capabilities(features: Any, components: frozenset[Component]) -> Capabilities: """The server's table, narrowed to what the calendar says it holds.""" table = {f: v for f, v in _table(features).items() if f not in BACKEND_FEATURES} @@ -289,14 +332,24 @@ def collection_capabilities(features: Any, components: frozenset[Component]) -> "lost": ["users"], "note": "the server refuses an event with a PARTICIPANT, which names who did the work", } + if Component.EVENT not in components and _level(features, "save-load.event.floating") is U: + # Probed on events, and SOGo keeps a task's floating times: a task list is not known to lose them + table[Feature.TIMEZONES_FLOATING] = {"level": X, "note": "floating times are probed on events only"} + if Component.EVENT not in components and _level(features, "save-load.event.recurrences.rdate") is U: + table[Feature.RECURRENCE_RDATE] = {"level": X, "note": "RDATE is probed on events only"} if Component.TASK in components: lowered = {**TASK_ONLY_FROM_SERVER, **(TASK_FROM_SERVER if Component.EVENT not in components else {})} for feature, name in lowered.items(): if _level(features, name) is U: table[feature] = U + if Component.EVENT not in components and (alarm := _alarm(features)) is not None: + table[Feature.ALARMS] = alarm for feature in VALUES_FROM_SERVER: # Ox keeps priorities 1, 5 and 9; Zimbra refuses CANCELLED if (declared := _values(features, feature)) is not None: table[feature] = declared + for feature, (name, needed, note) in REQUIRES_FROM_SERVER.items(): + if _level(features, name) is U: + table[feature] = {"level": L, "requires": needed, "note": note} stable_url = features.is_supported("save-load.stable-url", dict) if _hinted(features) else {} if stable_url.get("forces_uid_ics"): table[Feature.IDENTITY_CLIENT_NATIVE_ID] = {"level": U, "note": "the server renames every object .ics"} diff --git a/src/calendaring/_caldav_io.py b/src/calendaring/_caldav_io.py index b530e25..27e9c15 100644 --- a/src/calendaring/_caldav_io.py +++ b/src/calendaring/_caldav_io.py @@ -38,6 +38,7 @@ BackendError, CalendaringError, ConflictError, + ListingLimitError, NotFoundError, RateLimitError, SyncTokenError, @@ -109,6 +110,9 @@ def _translate(exc: dav_error.DAVError) -> CalendaringError: return ConflictError(message) if isinstance(exc, dav_error.AuthorizationError): return (AuthenticationError if exc.status == 401 else AuthorizationError)(message) + # A 507 inside a multistatus, or for a whole REPORT or PROPFIND: a listing the server cut. A PUT's is a full disk. + if isinstance(exc, (dav_error.ResponseError, dav_error.ReportError, dav_error.PropfindError)) and exc.status == 507: + return ListingLimitError(f"the server answered a listing with 507 Insufficient Storage: {message}") if isinstance(exc, dav_error.RateLimitError): return RateLimitError(message, retry_after=exc.retry_after_seconds) return BackendError(message) diff --git a/src/calendaring/_mapping.py b/src/calendaring/_mapping.py index 352913e..774dc19 100644 --- a/src/calendaring/_mapping.py +++ b/src/calendaring/_mapping.py @@ -12,12 +12,14 @@ import copy import functools import hashlib +import itertools import zoneinfo from collections.abc import Callable -from datetime import UTC, date, datetime, time -from typing import Any +from datetime import UTC, date, datetime, time, timedelta +from typing import Any, TypeVar import icalendar +import recurring_ical_events from icalendar.parser import Contentline from .capabilities import Capabilities, Feature, Support @@ -26,6 +28,8 @@ __all__ = ["Change", "differences", "plan_calendar_losses", "plan_losses", "property_lines"] +T = TypeVar("T") + # A loss and the edit that makes the component storable. Change = tuple[Loss, Callable[[], None]] @@ -44,6 +48,8 @@ CONVERSIONS: dict[Feature, Callable[[Any], Any]] = { # A time with a TZID, or a floating one (read as local time, as the searcher does), becomes UTC. Feature.TIMEZONES: lambda t: t.astimezone(UTC) if isinstance(t, datetime) and t.tzname() != "UTC" else t, + # A floating time only, read as local time; a TZID is kept (Zimbra, SOGo). + Feature.TIMEZONES_FLOATING: lambda t: t.astimezone(UTC) if isinstance(t, datetime) and t.tzinfo is None else t, # A date becomes midnight UTC. Feature.DATES: lambda t: ( t if isinstance(t, datetime) or not isinstance(t, date) else datetime.combine(t, time(), UTC) @@ -80,9 +86,19 @@ "RSVP": "FALSE", } +# Parameters a server may add without changing what a line means: an X- one anywhere, and on an +# attendee or organizer its name, address and scheduling state (SOGo adds RSVP, Ox EMAIL). +ADDABLE_PARAMETERS = { + "ATTENDEE": frozenset({"CN", "EMAIL", "RSVP", "SCHEDULE-AGENT", "SCHEDULE-STATUS", "SCHEDULE-FORCE-SEND"}), + "ORGANIZER": frozenset({"CN", "EMAIL", "SCHEDULE-AGENT", "SCHEDULE-STATUS", "SCHEDULE-FORCE-SEND"}), +} + # Single date-time properties a server may store in UTC instead of with their TZID. INSTANTS = frozenset({"DTSTART", "DTEND", "DUE", "COMPLETED", "RECURRENCE-ID", X_PLANNED_START, X_PLANNED_END}) +# Properties a server may add lines of without losing what was sent: the organizer as an attendee. +ADDED_LINES_KEEP_ALL = frozenset({"ATTENDEE"}) + # Properties servers legitimately change, and identity a backend may assign (§5.3). NOT_VERIFIED = frozenset({"DTSTAMP", "SEQUENCE", "LAST-MODIFIED", "UID"}) @@ -139,9 +155,12 @@ def apply() -> None: def _time_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: changes: list[Change] = [] + kept = (Support.FULL, Support.EMULATED, Support.UNKNOWN) for feature, convert in CONVERSIONS.items(): - if capabilities.level(feature) in (Support.FULL, Support.EMULATED, Support.UNKNOWN): + if capabilities.level(feature) in kept: continue + if feature is Feature.TIMEZONES_FLOATING and capabilities.level(Feature.TIMEZONES) not in kept: + continue # timezones converts floating times already for name in TIME_PROPERTIES: if name not in component: continue @@ -154,13 +173,17 @@ def _time_losses(component: icalendar.Component, capabilities: Capabilities) -> def _alarm_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: alarms = [c for c in component.subcomponents if c.name == "VALARM"] - if not alarms or capabilities.level(Feature.ALARMS) is not Support.UNSUPPORTED: + level = capabilities.level(Feature.ALARMS) + if not alarms or level not in (Support.UNSUPPORTED, Support.LOSSY): return [] + sent = alarms[0].to_ical().decode() + if level is Support.LOSSY: # rewritten in a way not modelled here (Ox): sent as it is + return [(Loss("VALARM", sent, sent, note=capabilities[Feature.ALARMS].note), lambda: None)] def apply() -> None: component.subcomponents[:] = [c for c in component.subcomponents if c.name != "VALARM"] - return [(Loss("VALARM", alarms[0].to_ical().decode(), None), apply)] + return [(Loss("VALARM", sent, None), apply)] def plan_losses(component: icalendar.Component, capabilities: Capabilities) -> list[Change]: @@ -189,6 +212,9 @@ def plan_losses(component: icalendar.Component, capabilities: Capabilities) -> l stored = mapping.get(sent, mapping.get(str(sent))) apply = _drop(component, name) if stored is None else _replace(component, name, stored) changes.append((Loss(name, sent, stored), apply)) + elif level is Support.LOSSY and (needed := capabilities.details(feature).get("requires")): + if not any(other in component for other in needed): # Zimbra: a task's DTSTART needs a DUE + changes.append((Loss(name, component[name], None), _drop(component, name))) if is_task and "RELATED-TO" in component: by_feature: dict[Feature, list[Any]] = {} for value in _values(component, "RELATED-TO"): @@ -202,6 +228,13 @@ def plan_losses(component: icalendar.Component, capabilities: Capabilities) -> l changes.append( (Loss("RELATED-TO", sent if len(sent) > 1 else sent[0], None), _keep_related(component, dropped)) ) + if ( + "RDATE" in component + and not is_task # probed on events, and a calendar's level is for both + and "RDATE" not in {loss.property for loss, _ in changes} + and capabilities.level(Feature.RECURRENCE_RDATE) is Support.UNSUPPORTED # Zimbra keeps an RRULE, not an RDATE + ): + changes.append((Loss("RDATE", component["RDATE"], None), _drop(component, "RDATE"))) dropped_names = {loss.property for loss, _ in changes} changes.extend(c for c in _time_losses(component, capabilities) if c[0].property not in dropped_names) return changes @@ -217,7 +250,7 @@ def _lines(component: icalendar.Component) -> list[bytes]: def plan_calendar_losses(calendar: icalendar.Calendar, capabilities: Capabilities) -> list[Change]: """The losses across a stored object's components: overrides go with the RRULE they belong to.""" if capabilities.level(Feature.RECURRENCE) is not Support.UNSUPPORTED: - return [] + return _rdate_override_losses(calendar, capabilities) overrides = [c for c in calendar.subcomponents if "RECURRENCE-ID" in c] if not overrides or len(overrides) == len([c for c in calendar.subcomponents if c.name != "VTIMEZONE"]): return [] # occurrences without a master are items of their own @@ -229,6 +262,39 @@ def apply() -> None: return [(Loss("RECURRENCE-ID", sent if len(sent) > 1 else sent[0], None), apply)] +def _rdate_override_losses(calendar: icalendar.Calendar, capabilities: Capabilities) -> list[Change]: + """An event's override of an occurrence only its RDATE gives goes with the RDATE (Zimbra): it would be an orphan.""" + if capabilities.level(Feature.RECURRENCE_RDATE) is not Support.UNSUPPORTED: + return [] + master = next((c for c in calendar.walk("VEVENT") if "RECURRENCE-ID" not in c and "RDATE" in c), None) + if master is None: + return [] + rrule_only = icalendar.Calendar() + rrule_only.add_component(without := copy.deepcopy(master)) + without.pop("RDATE") + + def rrule_gives(when: Any) -> bool: + return "RRULE" in master and any( # an occurrence starting there, not one merely overlapping it + o.decoded("DTSTART") == when + for o in recurring_ical_events.of(rrule_only).between(when, when + timedelta(seconds=1)) + ) + + rdates = {d[0] if isinstance(d, tuple) else d for d in _times(master, "RDATE")} # a PERIOD by its start + orphans = [ + c + for c in calendar.walk("VEVENT") + if "RECURRENCE-ID" in c and (when := c.decoded("RECURRENCE-ID")) in rdates and not rrule_gives(when) + ] + if not orphans: + return [] + + def apply() -> None: + calendar.subcomponents[:] = [c for c in calendar.subcomponents if not any(c is o for o in orphans)] + + sent = [c.get("RECURRENCE-ID").to_ical().decode() for c in orphans] + return [(Loss("RECURRENCE-ID", sent if len(sent) > 1 else sent[0], None), apply)] + + def property_lines(component: icalendar.Component, name: str) -> list[bytes]: """One property's content lines, unfolded and sorted, for an order-insensitive comparison. @@ -261,7 +327,7 @@ def _comparable(component: icalendar.Component, name: str) -> list[Any]: for one in str(value).split(",") } ) - if name in INSTANTS and not isinstance(component.get(name), list) and not _recurs(component): + if name in INSTANTS and name in component and not isinstance(component[name], list) and not _recurs(component): value = component.decoded(name) if isinstance(value, datetime) and value.tzinfo is not None: return [value.astimezone(UTC)] @@ -312,9 +378,7 @@ def _stored_as_sent(sent: icalendar.Component, stored: icalendar.Component) -> b """Whether ``stored`` holds everything ``sent`` did, as the top level is compared: what a server adds is fine.""" if sent.name != stored.name: return False - if any( - _comparable(sent, name) != _comparable(stored, name) for name in dict.fromkeys(sent) if name not in NOT_VERIFIED - ): + if not all(_kept(sent, stored, name) for name in dict.fromkeys(sent) if name not in NOT_VERIFIED): return False return _matched(sent.subcomponents, stored.subcomponents) @@ -325,14 +389,55 @@ def _matched(sent: list[icalendar.Component], stored: list[icalendar.Component]) Tried every way, not first come first served: a sent alarm that matches two stored ones must not take the one another sent alarm needs. A component has few. """ + return _paired(sent, stored, _stored_as_sent) + + +def _paired(sent: list[T], stored: list[T], fits: Callable[[T, T], bool]) -> bool: + """Each of ``sent`` ``fits`` one of ``stored`` of its own, tried every way.""" if not sent: return True first, rest = sent[0], sent[1:] return any( - _stored_as_sent(first, other) and _matched(rest, stored[:i] + stored[i + 1 :]) for i, other in enumerate(stored) + fits(first, other) and _paired(rest, stored[:i] + stored[i + 1 :], fits) for i, other in enumerate(stored) ) +def _kept(sent: icalendar.Component, stored: icalendar.Component, name: str) -> bool: + """Whether ``stored`` has ``name`` as ``sent`` had it. + + A parameter the server added that says nothing about the value (``_covered``) is no + difference. One it dropped or changed is, and so is a line less: a stored line holds one + sent line only. A line more is too, except an attendee that is the stored organizer: a + scheduling server adds the organizer as one (Ox). + """ + mine, theirs = _comparable(sent, name), _comparable(stored, name) + if mine == theirs: + return True + if not all(isinstance(line, tuple) for line in [*mine, *theirs]): + return False + if name in ADDED_LINES_KEEP_ALL and len(theirs) > len(mine): + organizer = str(stored.get("ORGANIZER", "")).lower() + extra = len(theirs) - len(mine) + candidates = [i for i, line in enumerate(theirs) if organizer and str(line[2]).lower() == organizer] + return any( + _paired(mine, [line for i, line in enumerate(theirs) if i not in left_out], _covered) + for left_out in itertools.combinations(candidates, extra) + ) + return len(mine) == len(theirs) and _paired(mine, theirs, _covered) + + +def _covered(sent: tuple[Any, ...], stored: tuple[Any, ...]) -> bool: + """Whether a stored line, as ``_without_defaults`` gives it, is the sent one with harmless parameters added. + + A parameter left out means its default, so one added with another value (PARTSTAT=DECLINED) + changes the line, and so does one that changes how the value reads (TZID, VALUE, RELTYPE). + """ + if (sent[0], sent[2]) != (stored[0], stored[2]) or not set(sent[1]) <= set(stored[1]): + return False + addable = ADDABLE_PARAMETERS.get(str(sent[0]), frozenset()) + return all(key.startswith("X-") or key in addable for key, _ in set(stored[1]) - set(sent[1])) + + def _subcomponents(component: icalendar.Component, kind: str) -> list[icalendar.Component]: return [c for c in component.subcomponents if c.name == kind] @@ -351,6 +456,22 @@ def _by_occurrence(calendar: icalendar.Calendar) -> dict[str | None, icalendar.C return out +def undeclared(found: list[Loss], capabilities: Capabilities) -> list[Loss]: + """The ``differences`` the declaration does not say a write may cause: a ``LOSSY`` alarm may come back changed. + + Changed, not gone: as many alarms must come back as were sent. + """ + if capabilities.level(Feature.ALARMS) is Support.LOSSY: + return [ + loss for loss in found if not (loss.property == "VALARM" and _alarms(loss.sent) == _alarms(loss.stored)) + ] + return found + + +def _alarms(text: object) -> int: + return str(text or "").count("BEGIN:VALARM") + + def differences(sent: icalendar.Calendar, stored: icalendar.Calendar) -> list[Loss]: """Every property that was sent and is stored differently, or not at all.""" out: list[Loss] = [] @@ -363,7 +484,7 @@ def differences(sent: icalendar.Calendar, stored: icalendar.Calendar) -> list[Lo for name in dict.fromkeys(component): if name in NOT_VERIFIED: continue - if _comparable(component, name) != _comparable(other, name): + if not _kept(component, other, name): mine, theirs = property_lines(component, name), property_lines(other, name) out.append(Loss(name, b"\n".join(mine).decode(), b"\n".join(theirs).decode() or None)) for kind in dict.fromkeys(str(c.name) for c in [*component.subcomponents, *other.subcomponents]): diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index 19e609e..d1bbb31 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -86,6 +86,8 @@ def _stored(self, obj: Any) -> StoredObject: # raw_data is what the server sent; data is what caldav made of it, which is what gets parsed. data = obj.data parsed = _parse(data.encode() if isinstance(data, str) else bytes(data)) + # CCS drops a VTIMEZONE, which RFC 5545 section 3.2.19 requires for each TZID; zoneinfo has it. + parsed.add_missing_timezones() return StoredObject(_name_of(obj.url), obj.etag, parsed, obj.raw_data, obj) def _listed(self, obj: Any) -> StoredObject | BrokenObject: diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index 4392044..b153eb1 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -25,7 +25,7 @@ import recurring_ical_events from icalendar_searcher import Searcher -from .._mapping import Change, differences, plan_calendar_losses, plan_losses +from .._mapping import NOT_VERIFIED, Change, differences, plan_calendar_losses, plan_losses, undeclared from ..capabilities import BACKEND_FEATURES, COMPONENT_FEATURE, Capabilities, Feature, Support from ..errors import ( AlreadyExistsError, @@ -93,6 +93,14 @@ def _occurrence_now(calendar: icalendar.Calendar, item: Item) -> icalendar.Compo return None +def _unstamped(component: icalendar.Component) -> icalendar.Component: + """``component`` without what a server rewrites in every component of an object it stores (Ox).""" + out = copy.deepcopy(component) + for name in NOT_VERIFIED: + out.pop(name, None) + return out + + def _occurs_at(item: Item) -> datetime: """Where an occurrence sorts: its ``RECURRENCE-ID`` as an aware instant (floating counts as UTC).""" rid = item.component.get("RECURRENCE-ID") @@ -431,7 +439,7 @@ def _prepare( def _check(self, sent: icalendar.Calendar, result: SyncItem) -> None: assert result.native_id is not None stored = self._fetch(result.native_id) - found = differences(sent, stored.calendar) + found = undeclared(differences(sent, stored.calendar), self.capabilities) if found: raise VerificationError(found) @@ -819,7 +827,8 @@ def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: Siblings share the stored object's etag, so a stale etag is not a conflict by itself: only a change to this occurrence on the server is. It is found by comparing the stored occurrence with the item as the - caller holds it, so once the etag moved, a local edit before + caller holds it, leaving out what verify never compares (DTSTAMP, SEQUENCE, + LAST-MODIFIED, UID), so once the etag moved, a local edit before ``delete()`` also counts as one; with the etag unchanged nothing changed on the server, and a local edit is deleted with the rest. An occurrence already gone is ``NotFoundError``. @@ -830,7 +839,7 @@ def _delete_occurrence(self, item: Item, *, overwrite: bool) -> None: now = _occurrence_now(stored.calendar, item) if now is None: raise NotFoundError(f"the occurrence {item.component.decoded('RECURRENCE-ID')} is gone") - if not overwrite and now != item.component: + if not overwrite and _unstamped(now) != _unstamped(item.component): raise ConflictError( f"the occurrence {item.component.decoded('RECURRENCE-ID')} changed since it was read" ) diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index 021c24c..f1377ad 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -47,6 +47,7 @@ async def open_backend(self) -> AsyncBackend: BackendError, CalendaringError, ConflictError, + ListingLimitError, LossyWriteError, LossyWriteWarning, NotFoundError, @@ -773,8 +774,8 @@ def test_property_round_trips_as_declared(self, collection: Collection, feature: assert support is Support.LOSSY, f"{feature} is declared {support.value} but the write is lossy" return read = collection.reload(stored) - if support is not Support.LOSSY: - assert property_lines(read.component, name) == property_lines(item.component, name) + if support is not Support.LOSSY: # stored as verify sees it: a parameter the server adds loses nothing + assert [d for d in differences(item.icalendar, read.icalendar) if d.property == name] == [] @pytest.mark.parametrize("feature", list(VALUE_SAMPLES)) def test_values_map_as_declared(self, collection: Collection, feature: Feature) -> None: @@ -921,6 +922,14 @@ def fields(items: Sequence[SyncItem]) -> list[tuple[Any, ...]]: def test_listings_are_not_cut_at_a_page(self, collection: Collection) -> None: kind = next(k for k in KINDS if holds(collection, k)) seeded = self.seed(collection, [minimal(kind, f"paged {n}") for n in range(self.page_size_hint)]) + try: + self._listings_are_complete(collection, seeded) + except ListingLimitError: + # Declared: the server cuts a long listing, and says so (Stalwart, at its WebDav.maxResults) + if level(collection, Feature.LISTING_COMPLETE) is not Support.UNSUPPORTED: + raise + + def _listings_are_complete(self, collection: Collection, seeded: list[SyncItem]) -> None: mine = {nid(i) for i in seeded} assert {nid(i) for i in stored_by(collection.search(), seeded)} == mine assert {nid(i) for i in collection.iter_search() if i.native_id in mine} == mine diff --git a/src/calendaring/capabilities.py b/src/calendaring/capabilities.py index e6183a2..0817d12 100644 --- a/src/calendaring/capabilities.py +++ b/src/calendaring/capabilities.py @@ -45,6 +45,7 @@ class Feature(StrEnum): SEARCH_SERVER_SIDE = "search.server-side" SEARCH_TIME_RANGE = "search.time-range" SEARCH_TEXT = "search.text" + LISTING_COMPLETE = "listing.complete" CHANGES = "changes" CHANGES_KNOWN = "changes.known" WRITE_WIRE = "write.wire" @@ -54,6 +55,7 @@ class Feature(StrEnum): IDENTITY_CLIENT_NATIVE_ID = "identity.client-native-id" PROPERTIES_PASSTHROUGH = "properties.passthrough" RECURRENCE = "recurrence" + RECURRENCE_RDATE = "recurrence.rdate" RECURRENCE_EDIT_OCCURRENCE = "recurrence.edit-occurrence" RECURRENCE_EDIT_THIS_AND_FUTURE = "recurrence.edit-this-and-future" MOVE = "move" @@ -73,6 +75,7 @@ class Feature(StrEnum): ATTENDEES = "attendees" ALARMS = "alarms" TIMEZONES = "timezones" + TIMEZONES_FLOATING = "timezones.floating" DATES = "dates" TIME_LOG = "time-log" diff --git a/src/calendaring/errors.py b/src/calendaring/errors.py index 0f0c770..3593aec 100644 --- a/src/calendaring/errors.py +++ b/src/calendaring/errors.py @@ -26,6 +26,7 @@ "ConfigurationError", "ConflictError", "InvalidDataError", + "ListingLimitError", "LossyWriteError", "LossyWriteWarning", "NotFoundError", @@ -198,6 +199,15 @@ class TransportError(BackendError): """Network, TLS, timeout. During a write, the outcome is unknown.""" +class ListingLimitError(BackendError): + """The server cut a listing at a limit of its own, and said so with a 507; none of it is returned. + + Where ``listing.complete`` is ``UNSUPPORTED`` (Stalwart, at its + ``WebDav.maxResults``). A server that cuts a listing without a 507 is + ``UNSUPPORTED`` too, and its listing comes back short instead. + """ + + class CalendaringWarning(UserWarning): """The base of this library's warnings.""" diff --git a/src/calendaring/loss.py b/src/calendaring/loss.py index 84fe60c..84c115c 100644 --- a/src/calendaring/loss.py +++ b/src/calendaring/loss.py @@ -27,6 +27,7 @@ class Loss: property: str # e.g. "PRIORITY", "STATUS", "RELATED-TO" sent: object # the value in the item stored: object | None # what the backend will store instead; None if dropped + note: str | None = None # how the server changes it, where the backend cannot say what it stores: stored is sent class LossPolicy(Protocol): @@ -38,7 +39,9 @@ def _raise(losses: Sequence[Loss], item: Item) -> None: def _warn(losses: Sequence[Loss], item: Item) -> None: - text = ", ".join(f"{loss.property}={loss.sent!r} -> {loss.stored!r}" for loss in losses) + text = ", ".join( + f"{loss.property}={loss.sent!r} -> {loss.note if loss.note else repr(loss.stored)}" for loss in losses + ) warnings.warn(f"lossy write of {item.uid}: {text}", LossyWriteWarning, stacklevel=2) diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index 7edf749..00deda8 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -6,8 +6,11 @@ import base64 import dataclasses from collections.abc import AsyncIterator +from datetime import datetime from typing import Any +from zoneinfo import ZoneInfo +import icalendar import pytest from calendaring import ( @@ -199,6 +202,29 @@ async def spy(url: str, body: bytes, headers: Any = None) -> Any: assert Task.from_ical(reread.wire) == reread +async def test_a_dropped_vtimezone_is_put_back_on_read( + collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch +) -> None: + """CCS stores TZID=Europe/Oslo but not its VTIMEZONE, which RFC 5545 section 3.2.19 requires.""" + client = collection._backend._dav.client # type: ignore[attr-defined] + put = client.put + + async def dropping(url: str, body: bytes, headers: Any = None) -> Any: + calendar = icalendar.Calendar.from_ical(body) + calendar.subcomponents = [c for c in calendar.subcomponents if c.name != "VTIMEZONE"] + return await put(url, calendar.to_ical(), headers) + + monkeypatch.setattr(client, "put", dropping) + due = datetime(2026, 11, 20, 10, tzinfo=ZoneInfo("Europe/Oslo")) + stored = await collection.add_task(summary="zoned", end=due) + read = await collection.get_by_native_id(stored.native_id or "") + assert read.wire is not None + assert b"VTIMEZONE" not in read.wire + assert [str(tz["TZID"]) for tz in read.icalendar.walk("VTIMEZONE")] == ["Europe/Oslo"] + assert isinstance(read.component, icalendar.Todo) + assert read.component.end == due + + def weakened(collection: AsyncCollection, monkeypatch: pytest.MonkeyPatch) -> None: """Make every ETag read from the server weak, as GMX hands them out.""" stored = collection._stored # type: ignore[attr-defined] diff --git a/tests/_async/test_caldav_conformance.py b/tests/_async/test_caldav_conformance.py index 6a9fc6b..71dafca 100644 --- a/tests/_async/test_caldav_conformance.py +++ b/tests/_async/test_caldav_conformance.py @@ -6,7 +6,7 @@ from typing import Any import pytest -from conftest import wait_out_search_cache +from conftest import wait_for_server from calendaring import AsyncBackend from calendaring.testing import AsyncConformance @@ -20,7 +20,7 @@ class CalDAVConformance(AsyncConformance): async def open_backend(self) -> AsyncBackend: backend = await AsyncBackend.open(self.server.url, kind="caldav", **self.server.options) - wait_out_search_cache(backend, self.monkeypatch) + wait_for_server(backend, self.monkeypatch) return backend diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index 94948c7..9917ed9 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -161,6 +161,39 @@ async def test_reload_finds_a_moved_override() -> None: assert (await collection.reload(generated)).component.start == at(2, 9) +class Restamps(AsyncMemoryCollection): + """A server stamping every component of an object it rewrites (Ox).""" + + async def _update( + self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None + ) -> StoredObject: + for component in calendar.walk("VEVENT"): + for name in ("DTSTAMP", "LAST-MODIFIED"): + component.pop(name, None) + component.add(name, datetime.now(UTC) + timedelta(minutes=1)) + return await super()._update(native_id, calendar, etag=etag, wire=None) + + +async def test_a_sibling_restamped_by_the_server_has_not_changed() -> None: + class Backend(AsyncMemoryBackend): + collection_class = Restamps + + collection = await Backend().collection("default") + series = Event.new(summary="daily", start=at(1, 9), end=at(1, 10)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) + moved = Event.new(summary="moved", start=at(3, 15), end=at(3, 16), uid=series.uid) + moved.component.add("RECURRENCE-ID", at(3, 9)) + series.icalendar.add_component(moved.component) + await collection.add(series) + found = await collection.search(event=True, start=at(1, 0), end=at(6, 0), expand=True) + by_day = {i.component.decoded("RECURRENCE-ID").day: i for i in found} + await collection.delete(by_day[2]) + await collection.delete(by_day[3]) # the override, its DTSTAMP and LAST-MODIFIED rewritten + by_day[4].component["SUMMARY"] = "edited elsewhere" + with pytest.raises(ConflictError): + await collection.delete(by_day[4], overwrite=False) + + async def test_a_collection_narrows_a_backend_level_entry() -> None: backend = AsyncMemoryBackend() backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.FULL}) diff --git a/tests/_async/test_memory_conformance.py b/tests/_async/test_memory_conformance.py index 04458af..e736da5 100644 --- a/tests/_async/test_memory_conformance.py +++ b/tests/_async/test_memory_conformance.py @@ -36,6 +36,7 @@ Component, Feature, Item, + ListingLimitError, NotFoundError, Support, SyncToken, @@ -399,6 +400,29 @@ async def _list_etags(self) -> dict[str, str]: return {o.native_id: o.etag for o in self._page()} +class CutsWithAnError(FirstPageOnly): + """Stalwart's way: a listing longer than the server's limit fails, with nothing of it returned.""" + + def _page(self) -> list[MemoryObject]: + if len(self.store.objects) > 50: + raise ListingLimitError("the server cut the listing at 50") + return super()._page() + + +class DeclaresItsLimit(CutsWithAnError): + def __init__(self, backend: AsyncMemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.LISTING_COMPLETE: Support.UNSUPPORTED}) + + +class DeclaresAndCutsSilently(FirstPageOnly): + """Declares the limit, but hands out the first page as if it were all.""" + + def __init__(self, backend: AsyncMemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.LISTING_COMPLETE: Support.UNSUPPORTED}) + + class MatchesUidSubstrings(AsyncMemoryCollection): """Looks a UID up with a server text-match, newest first.""" @@ -630,6 +654,8 @@ async def _time_entries( CAUGHT: list[tuple[type[AsyncMemoryCollection], str, dict[str, Any], Any]] = [ (FirstPageOnly, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), + (CutsWithAnError, "test_listings_are_not_cut_at_a_page", {}, ListingLimitError), + (DeclaresAndCutsSilently, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), (MatchesUidSubstrings, "test_get_finds_the_exact_uid", {}, ASSERTS), (SearchesTheAccount, "test_get_finds_the_exact_uid", {}, ASSERTS), (ReplacesEventUids, "test_get_finds_the_exact_uid", {}, ASSERTS), @@ -695,3 +721,13 @@ async def test_the_suite_catches_a_collection_wider_than_its_backend() -> None: backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) with pytest.raises(AssertionError): await TestFullProfile().test_collection_never_widens_backend(backend, await backend.collection("default")) + + +async def test_the_suite_accepts_a_declared_listing_limit() -> None: + class Backend(AsyncMemoryBackend): + collection_class = DeclaresItsLimit + + suite = TestFullProfile() + suite._made, suite._borrowed = [], [] + backend = Backend(profile=FULL) + await suite.test_listings_are_not_cut_at_a_page(await suite.open_collection(backend)) diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index eefa38f..2527e27 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -6,8 +6,11 @@ import base64 import dataclasses from collections.abc import Iterator +from datetime import datetime from typing import Any +from zoneinfo import ZoneInfo +import icalendar import pytest from calendaring import ( @@ -199,6 +202,29 @@ def spy(url: str, body: bytes, headers: Any = None) -> Any: assert Task.from_ical(reread.wire) == reread +def test_a_dropped_vtimezone_is_put_back_on_read( + collection: Collection, monkeypatch: pytest.MonkeyPatch +) -> None: + """CCS stores TZID=Europe/Oslo but not its VTIMEZONE, which RFC 5545 section 3.2.19 requires.""" + client = collection._backend._dav.client # type: ignore[attr-defined] + put = client.put + + def dropping(url: str, body: bytes, headers: Any = None) -> Any: + calendar = icalendar.Calendar.from_ical(body) + calendar.subcomponents = [c for c in calendar.subcomponents if c.name != "VTIMEZONE"] + return put(url, calendar.to_ical(), headers) + + monkeypatch.setattr(client, "put", dropping) + due = datetime(2026, 11, 20, 10, tzinfo=ZoneInfo("Europe/Oslo")) + stored = collection.add_task(summary="zoned", end=due) + read = collection.get_by_native_id(stored.native_id or "") + assert read.wire is not None + assert b"VTIMEZONE" not in read.wire + assert [str(tz["TZID"]) for tz in read.icalendar.walk("VTIMEZONE")] == ["Europe/Oslo"] + assert isinstance(read.component, icalendar.Todo) + assert read.component.end == due + + def weakened(collection: Collection, monkeypatch: pytest.MonkeyPatch) -> None: """Make every ETag read from the server weak, as GMX hands them out.""" stored = collection._stored # type: ignore[attr-defined] diff --git a/tests/_sync/test_caldav_conformance.py b/tests/_sync/test_caldav_conformance.py index 0964345..2a3ca0b 100644 --- a/tests/_sync/test_caldav_conformance.py +++ b/tests/_sync/test_caldav_conformance.py @@ -6,7 +6,7 @@ from typing import Any import pytest -from conftest import wait_out_search_cache +from conftest import wait_for_server from calendaring import Backend from calendaring.testing import SyncConformance @@ -20,7 +20,7 @@ class CalDAVConformance(SyncConformance): def open_backend(self) -> Backend: backend = Backend.open(self.server.url, kind="caldav", **self.server.options) - wait_out_search_cache(backend, self.monkeypatch) + wait_for_server(backend, self.monkeypatch) return backend diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 0db6252..3cf3cde 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -161,6 +161,39 @@ def test_reload_finds_a_moved_override() -> None: assert (collection.reload(generated)).component.start == at(2, 9) +class Restamps(MemoryCollection): + """A server stamping every component of an object it rewrites (Ox).""" + + def _update( + self, native_id: str, calendar: icalendar.Calendar, *, etag: str | None, wire: bytes | None + ) -> StoredObject: + for component in calendar.walk("VEVENT"): + for name in ("DTSTAMP", "LAST-MODIFIED"): + component.pop(name, None) + component.add(name, datetime.now(UTC) + timedelta(minutes=1)) + return super()._update(native_id, calendar, etag=etag, wire=None) + + +def test_a_sibling_restamped_by_the_server_has_not_changed() -> None: + class Backend(MemoryBackend): + collection_class = Restamps + + collection = Backend().collection("default") + series = Event.new(summary="daily", start=at(1, 9), end=at(1, 10)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 5}) + moved = Event.new(summary="moved", start=at(3, 15), end=at(3, 16), uid=series.uid) + moved.component.add("RECURRENCE-ID", at(3, 9)) + series.icalendar.add_component(moved.component) + collection.add(series) + found = collection.search(event=True, start=at(1, 0), end=at(6, 0), expand=True) + by_day = {i.component.decoded("RECURRENCE-ID").day: i for i in found} + collection.delete(by_day[2]) + collection.delete(by_day[3]) # the override, its DTSTAMP and LAST-MODIFIED rewritten + by_day[4].component["SUMMARY"] = "edited elsewhere" + with pytest.raises(ConflictError): + collection.delete(by_day[4], overwrite=False) + + def test_a_collection_narrows_a_backend_level_entry() -> None: backend = MemoryBackend() backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.FULL}) diff --git a/tests/_sync/test_memory_conformance.py b/tests/_sync/test_memory_conformance.py index e47c54a..e8a259d 100644 --- a/tests/_sync/test_memory_conformance.py +++ b/tests/_sync/test_memory_conformance.py @@ -36,6 +36,7 @@ Component, Feature, Item, + ListingLimitError, NotFoundError, Support, SyncToken, @@ -399,6 +400,29 @@ def _list_etags(self) -> dict[str, str]: return {o.native_id: o.etag for o in self._page()} +class CutsWithAnError(FirstPageOnly): + """Stalwart's way: a listing longer than the server's limit fails, with nothing of it returned.""" + + def _page(self) -> list[MemoryObject]: + if len(self.store.objects) > 50: + raise ListingLimitError("the server cut the listing at 50") + return super()._page() + + +class DeclaresItsLimit(CutsWithAnError): + def __init__(self, backend: MemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.LISTING_COMPLETE: Support.UNSUPPORTED}) + + +class DeclaresAndCutsSilently(FirstPageOnly): + """Declares the limit, but hands out the first page as if it were all.""" + + def __init__(self, backend: MemoryBackend, store: MemoryCalendar) -> None: + super().__init__(backend, store) + self.capabilities = self.capabilities.updated({Feature.LISTING_COMPLETE: Support.UNSUPPORTED}) + + class MatchesUidSubstrings(MemoryCollection): """Looks a UID up with a server text-match, newest first.""" @@ -630,6 +654,8 @@ def _time_entries( CAUGHT: list[tuple[type[MemoryCollection], str, dict[str, Any], Any]] = [ (FirstPageOnly, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), + (CutsWithAnError, "test_listings_are_not_cut_at_a_page", {}, ListingLimitError), + (DeclaresAndCutsSilently, "test_listings_are_not_cut_at_a_page", {}, ASSERTS), (MatchesUidSubstrings, "test_get_finds_the_exact_uid", {}, ASSERTS), (SearchesTheAccount, "test_get_finds_the_exact_uid", {}, ASSERTS), (ReplacesEventUids, "test_get_finds_the_exact_uid", {}, ASSERTS), @@ -695,3 +721,13 @@ def test_the_suite_catches_a_collection_wider_than_its_backend() -> None: backend.capabilities = backend.capabilities.updated({Feature.WRITE: Support.UNSUPPORTED}) with pytest.raises(AssertionError): TestFullProfile().test_collection_never_widens_backend(backend, backend.collection("default")) + + +def test_the_suite_accepts_a_declared_listing_limit() -> None: + class Backend(MemoryBackend): + collection_class = DeclaresItsLimit + + suite = TestFullProfile() + suite._made, suite._borrowed = [], [] + backend = Backend(profile=FULL) + suite.test_listings_are_not_cut_at_a_page(suite.open_collection(backend)) diff --git a/tests/conftest.py b/tests/conftest.py index b7a32df..37a0e29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -151,21 +151,31 @@ def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item item.add_marker(pytest.mark.xdist_group("Radicale")) -def wait_out_search_cache(backend: Any, monkeypatch: pytest.MonkeyPatch) -> None: - """A search on ``backend`` waits for what is left of its server's search-cache delay since a write. - - Bedework 3 and Purelymail answer from an index that lags behind writes; a user - seldom searches within seconds of a write, a story always does. caldav's own - integration tests wait the same way, with the same helper, which caldav ships - from 3.8; with an older caldav the stories on those servers may fail. +def wait_for_server(backend: Any, monkeypatch: pytest.MonkeyPatch) -> None: + """``backend``'s writes and searches wait for its server, as caldav's integration tests do. + + A read of an object waits out the server's ``synchronous-write`` delay since + it was written (Bedework 5 makes an object readable a few seconds after the + PUT is answered), and so does a search or listing where the server has no + search-cache delay of its own. A search waits + for what is left of the search-cache delay since a write: Bedework 3 and + Purelymail answer from an index that lags behind writes. A user seldom reads + within seconds of a write, a test always does. The search-cache helper comes + with caldav 3.8; with an older caldav the stories on those servers may fail. """ + _settle_writes(backend._dav.client, monkeypatch) try: from caldav.collection import Calendar from caldav.testing.search_cache_delay import SearchCacheDelay # type: ignore[import-not-found,unused-ignore] # caldav 3.8 except ImportError: return + from caldav.compatibility_hints import write_delay + client = backend._dav.client delay = SearchCacheDelay.from_features(client.features) + if delay is None and write_delay(client.features): + # Bedework 5 lags in searches and listings too, by the same few seconds + delay = SearchCacheDelay(write_delay(client.features)) if delay is None: return if inspect.iscoroutinefunction(client.request): @@ -179,6 +189,51 @@ def wait_out_search_cache(backend: Any, monkeypatch: pytest.MonkeyPatch) -> None monkeypatch.setattr(client, "propfind", _waiting(delay, client.propfind)) +def _settle_writes(client: Any, monkeypatch: pytest.MonkeyPatch) -> None: + """A read of an object through ``client`` waits out the server's ``synchronous-write`` delay since it was written. + + caldav's integration tests sleep the delay after every write, which makes a + run on Bedework 5 take hours. Here only a GET of a recently written object, + or a REPORT naming it (the multiget a read-back makes), waits. + """ + from urllib.parse import urlsplit + + from caldav.compatibility_hints import write_delay + + delay = write_delay(client.features) + if not delay: + return + request = client.request + written: dict[str, float] = {} + writes = {"PUT", "DELETE", "MKCALENDAR", "MKCOL", "PROPPATCH", "MOVE", "COPY", "POST"} + + def before(url: Any, method: str, body: Any) -> float: + """Seconds to wait before this request; a write is noted instead.""" + path = urlsplit(str(url)).path + now = time.monotonic() + if str(method).upper() in writes: + written[path] = now + return 0.0 + text = body.decode() if isinstance(body, bytes) else str(body or "") + due = [t + delay for p, t in written.items() if p == path or p in text] + return max([0.0, *(d - now for d in due)]) + + if inspect.iscoroutinefunction(request): + + async def settled(url: Any, method: str = "GET", body: Any = "", *args: Any, **kwargs: Any) -> Any: + await asyncio.sleep(before(url, method, body)) + return await request(url, method, body, *args, **kwargs) + + monkeypatch.setattr(client, "request", settled) + else: + + def settled_sync(url: Any, method: str = "GET", body: Any = "", *args: Any, **kwargs: Any) -> Any: + time.sleep(before(url, method, body)) + return request(url, method, body, *args, **kwargs) + + monkeypatch.setattr(client, "request", settled_sync) + + def _waiting(delay: Any, method: Any) -> Any: """``method``, after what is left of ``delay`` since the last write.""" if inspect.iscoroutinefunction(method): diff --git a/tests/test_caldav_features.py b/tests/test_caldav_features.py index f02f7ae..50e3ea3 100644 --- a/tests/test_caldav_features.py +++ b/tests/test_caldav_features.py @@ -88,6 +88,80 @@ def test_time_zones_follow_the_servers_hint() -> None: assert collection_capabilities(server, ALL).level(Feature.TIMEZONES) is Support.UNSUPPORTED +def test_floating_times_follow_the_servers_hint() -> None: + server = features({"save-load": {"support": "full"}, "save-load.event.floating": {"support": "unsupported"}}) + table = collection_capabilities(server, ALL) + assert table.level(Feature.TIMEZONES_FLOATING) is Support.LOSSY + assert table.level(Feature.TIMEZONES) is Support.FULL, "a TZID is kept" + assert collection_capabilities(hinted(save_load="full"), ALL).level(Feature.TIMEZONES_FLOATING) is Support.FULL + # The hint is probed on events: SOGo keeps a task's floating times, so a task list is not told otherwise + tasks = collection_capabilities(server, frozenset({Component.TASK})) + assert tasks.level(Feature.TIMEZONES_FLOATING) is Support.UNKNOWN + + +@pytest.mark.parametrize("hint", ["search.limit", "propfind.limit", "multiget.limit"]) +def test_a_listing_limit_makes_listings_incomplete(hint: str) -> None: + """Stalwart cuts each of the three at its WebDav.maxResults, with a 507.""" + server = features({"save-load": {"support": "full"}, hint: {"support": "unsupported"}}) + assert collection_capabilities(server, ALL).level(Feature.LISTING_COMPLETE) is Support.UNSUPPORTED + assert collection_capabilities(hinted(save_load="full"), ALL).level(Feature.LISTING_COMPLETE) is Support.FULL + + +@needs_hint("save-load.event.recurrences.rdate") +def test_rdates_follow_the_servers_hint() -> None: + """Zimbra drops every RDATE; probed on events, so a task list is not told so.""" + server = features( + {"save-load": {"support": "full"}, "save-load.event.recurrences.rdate": {"support": "unsupported"}} + ) + assert collection_capabilities(server, ALL).level(Feature.RECURRENCE_RDATE) is Support.UNSUPPORTED + assert collection_capabilities(server, ALL).level(Feature.RECURRENCE) is Support.FULL + tasks = collection_capabilities(server, frozenset({Component.TASK})) + assert tasks.level(Feature.RECURRENCE_RDATE) is Support.UNKNOWN + assert collection_capabilities(hinted(save_load="full"), ALL).level(Feature.RECURRENCE_RDATE) is Support.FULL + + +@needs_hint("save-load.todo.dtstart-without-due") +def test_a_task_start_kept_only_with_a_due() -> None: + server = features( + {"save-load": {"support": "full"}, "save-load.todo.dtstart-without-due": {"support": "unsupported"}} + ) + table = collection_capabilities(server, frozenset({Component.TASK})) + assert table.level(Feature.TASK_START) is Support.LOSSY + assert table.details(Feature.TASK_START)["requires"] == ["DUE"] + assert collection_capabilities(hinted(save_load="full"), ALL).level(Feature.TASK_START) is Support.FULL + + +def test_a_completed_kept_only_with_a_status() -> None: + """Ox drops COMPLETED from a task without a STATUS.""" + server = features( + {"save-load": {"support": "full"}, "save-load.todo.completed-without-status": {"support": "unsupported"}} + ) + table = collection_capabilities(server, frozenset({Component.TASK})) + assert table.level(Feature.TASK_COMPLETED) is Support.LOSSY + assert table.details(Feature.TASK_COMPLETED)["requires"] == ["STATUS"] + assert collection_capabilities(hinted(save_load="full"), ALL).level(Feature.TASK_COMPLETED) is Support.FULL + + +@needs_hint("save-load.todo.alarm") +def test_a_task_alarm_rewritten_is_lossy() -> None: + """Ox makes a task's alarm trigger absolute and replaces its description; an event's it keeps.""" + behaviour = "the alarm's TRIGGER -PT15M is stored as VALUE=DATE-TIME:20270120T114500Z" + server = features( + {"save-load": {"support": "full"}, "save-load.todo.alarm": {"support": "broken", "behaviour": behaviour}} + ) + tasks = collection_capabilities(server, frozenset({Component.TASK})) + assert tasks.level(Feature.ALARMS) is Support.LOSSY + assert behaviour in (tasks[Feature.ALARMS].note or "") + both = collection_capabilities(server, frozenset({Component.TASK, Component.EVENT})) + assert both.level(Feature.ALARMS) is Support.FULL + + +@needs_hint("save-load.todo.alarm") +def test_a_task_alarm_dropped_is_unsupported() -> None: + server = features({"save-load": {"support": "full"}, "save-load.todo.alarm": {"support": "unsupported"}}) + assert collection_capabilities(server, frozenset({Component.TASK})).level(Feature.ALARMS) is Support.UNSUPPORTED + + def test_a_calendar_without_a_component_cannot_hold_it() -> None: table = collection_capabilities(hinted(save_load="full"), frozenset({Component.TASK})) assert table.level(Feature.COMPONENT_TASK) is Support.FULL @@ -368,6 +442,11 @@ def test_a_name_is_checked_before_the_put_where_the_server_ignores_if_none_match assert ( collection_capabilities(hinted(save_load="full"), ALL).level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.FULL ) + # Bedework refuses a taken name with 500, not 412: caldav GETs the name first there too + mishandling = features( + {"save-load": {"support": "full"}, "save.if-none-match": {"support": "ungraceful"}, **refusing} + ) + assert collection_capabilities(mishandling, ALL).level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.EMULATED # Zimbra ignores it too, but renames every object: no native id at all renaming = features( { diff --git a/tests/test_caldav_io.py b/tests/test_caldav_io.py index 4ac131b..b341052 100644 --- a/tests/test_caldav_io.py +++ b/tests/test_caldav_io.py @@ -56,6 +56,22 @@ def test_a_401_is_authentication_and_a_403_authorization() -> None: assert type(_translate(dav_error.AuthorizationError(url="x", status=403))) is AuthorizationError +def test_a_507_in_a_listing_is_a_cut_listing() -> None: + """Stalwart lists up to its WebDav.maxResults and adds a 507 for the collection; + caldav's search.limit hint names a 507 for the whole query too.""" + from caldav.lib import error as dav_error + + from calendaring._caldav_io import _translate + from calendaring.errors import BackendError, ListingLimitError + + cut = dav_error.ResponseError("HTTP/1.1 507 Insufficient Storage", status=507) + assert type(_translate(cut)) is ListingLimitError + for whole in (dav_error.ReportError(url="x", status=507), dav_error.PropfindError(url="x", status=507)): + assert type(_translate(whole)) is ListingLimitError, "a 507 for the whole query" + assert type(_translate(dav_error.ResponseError("HTTP/1.1 500 X", status=500))) is BackendError + assert type(_translate(dav_error.PutError(url="x", status=507))) is BackendError, "a full disk, not a listing" + + def test_a_refused_create_says_what_was_taken() -> None: from caldav.lib import error as dav_error diff --git a/tests/test_capabilities.py b/tests/test_capabilities.py index b163c76..d24572e 100644 --- a/tests/test_capabilities.py +++ b/tests/test_capabilities.py @@ -68,6 +68,14 @@ def test_stock_loss_policies() -> None: assert set(BY_NAME) == {"raise", "warn", "allow"} +def test_a_loss_the_backend_cannot_predict_carries_a_note() -> None: + """Ox rewrites a task's alarm: what it stores is not known before, so the loss says how instead.""" + loss = Loss("VALARM", "BEGIN:VALARM...", "BEGIN:VALARM...", note="the trigger is made absolute") + assert Loss("PRIORITY", 3, 1).note is None + with pytest.warns(LossyWriteWarning, match="the trigger is made absolute"): + WARN([loss], Task.new(summary="x")) + + def test_errors_are_also_the_builtins() -> None: assert issubclass(NotFoundError, LookupError) assert issubclass(InvalidDataError, ValueError) diff --git a/tests/test_mapping.py b/tests/test_mapping.py index b85dac0..94404a2 100644 --- a/tests/test_mapping.py +++ b/tests/test_mapping.py @@ -4,12 +4,14 @@ import copy from datetime import UTC, date, datetime, timedelta +from typing import Any from zoneinfo import ZoneInfo import icalendar +import pytest from calendaring import Capabilities, Event, Feature, Support, Task, TaskStatus -from calendaring._mapping import differences, plan_calendar_losses, plan_losses +from calendaring._mapping import differences, plan_calendar_losses, plan_losses, undeclared from calendaring.memory import TRACKER OSLO = ZoneInfo("Europe/Oslo") @@ -48,6 +50,39 @@ def test_alarms_unsupported_drops_them() -> None: assert component.subcomponents == [] +def test_alarms_lossy_is_a_loss_and_sent_as_they_are() -> None: + """Ox rewrites a task's alarm in a way the backend does not model: it is sent as it is.""" + event = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC))) + sent = event.component.subcomponents[0].to_ical().decode() + table = {**dict.fromkeys(Feature, Support.FULL), Feature.ALARMS: {"level": Support.LOSSY, "note": "rewritten"}} + [(loss, apply)] = plan_losses(event.component, Capabilities(table)) + apply() + assert (loss.property, loss.sent, loss.stored, loss.note) == ("VALARM", sent, sent, "rewritten") + assert [c.name for c in event.component.subcomponents] == ["VALARM"] + + +def test_a_lossy_alarm_rewritten_is_no_undeclared_difference() -> None: + sent = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC))) + stored = copy.deepcopy(sent.icalendar) + for alarm in stored.walk("VALARM"): + alarm["DESCRIPTION"] = "Open-XChange" + found = differences(sent.icalendar, stored) + table = dict.fromkeys(Feature, Support.FULL) + assert undeclared(found, Capabilities(table)) == found + assert undeclared(found, Capabilities({**table, Feature.ALARMS: Support.LOSSY})) == [] + + +def test_a_lossy_alarm_dropped_is_still_an_undeclared_difference() -> None: + """LOSSY alarms may come back changed (Ox), not be gone.""" + sent = with_alarm(Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC))) + stored = copy.deepcopy(sent.icalendar) + for event in stored.walk("VEVENT"): + event.subcomponents = [] + found = differences(sent.icalendar, stored) + lossy = Capabilities({**dict.fromkeys(Feature, Support.FULL), Feature.ALARMS: Support.LOSSY}) + assert [loss.property for loss in undeclared(found, lossy)] == ["VALARM"] + + def test_timezones_lossy_stores_utc() -> None: event = Event.new( summary="x", start=datetime(2026, 11, 3, 9, tzinfo=OSLO), end=datetime(2026, 11, 3, 10, tzinfo=OSLO) @@ -62,6 +97,17 @@ def test_timezones_lossy_stores_utc() -> None: assert planned(event, {Feature.TIMEZONES: Support.FULL})[0] == [] +def test_floating_lossy_stores_floating_times_in_utc_only() -> None: + """Zimbra and SOGo keep a TZID but store a floating time in UTC.""" + zoned = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=OSLO)) + floating = Event.new(summary="y", start=datetime(2026, 11, 3, 9)) + assert planned(zoned, {Feature.TIMEZONES_FLOATING: Support.LOSSY})[0] == [] + losses, component = planned(floating, {Feature.TIMEZONES_FLOATING: Support.LOSSY}) + assert losses == ["DTSTART"] + assert component.decoded("DTSTART").tzinfo == UTC + assert planned(floating, {Feature.TIMEZONES_FLOATING: Support.FULL})[0] == [] + + def test_dates_lossy_stores_midnight_utc() -> None: event = Event.new(summary="x", start=date(2026, 11, 3), end=date(2026, 11, 4)) event.component.add("EXDATE", [date(2026, 11, 10)]) @@ -117,8 +163,60 @@ def test_a_parameter_added_with_its_default_is_no_difference() -> None: "ATTENDEE;CUTYPE=individual;PARTSTAT=NEEDS-ACTION;ROLE=REQ-PARTICIPANT;CN=Bob;RSVP=FALSE:mailto:bob@example.com" ) assert differences(sent.icalendar, stored_as(sent, defaults)) == [] - rsvp = "ATTENDEE;CN=Bob;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:mailto:bob@example.com" - assert [d.property for d in differences(sent.icalendar, stored_as(sent, rsvp))] == ["ATTENDEE"] + + +def test_a_parameter_the_server_added_is_no_difference() -> None: + sent = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + sent.component.add("ATTENDEE", "mailto:bob@example.com", parameters={"CN": "Bob"}) + for added in ( + "ATTENDEE;CN=Bob;PARTSTAT=NEEDS-ACTION;RSVP=TRUE:mailto:bob@example.com", # SOGo + "ATTENDEE;CN=Bob;EMAIL=bob@example.com:mailto:bob@example.com", # Ox + ): + assert differences(sent.icalendar, stored_as(sent, added)) == [] + for changed in ( + "ATTENDEE:mailto:bob@example.com", + "ATTENDEE;CN=Robert;RSVP=TRUE:mailto:bob@example.com", + "ATTENDEE;CN=Bob:mailto:robert@example.com", + ): + assert [d.property for d in differences(sent.icalendar, stored_as(sent, changed))] == ["ATTENDEE"] + + +def test_a_parameter_that_changes_the_meaning_is_a_difference_when_added() -> None: + """Only parameters that say nothing about the value may be added: a TZID, PARTSTAT or RELTYPE may.""" + series = Event.new(summary="x", start=datetime(2026, 11, 3, 9)) + series.component.add("RRULE", {"FREQ": "DAILY", "COUNT": 3}) + zoned = ( + series.icalendar.to_ical() + .decode() + .replace("DTSTART:20261103T090000", "DTSTART;TZID=Europe/Oslo:20261103T090000") + ) + assert [d.property for d in differences(series.icalendar, icalendar.Calendar.from_ical(zoned))] == ["DTSTART"] + meeting = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + meeting.component.add("ATTENDEE", "mailto:bob@example.com", parameters={"CN": "Bob"}) + declined = "ATTENDEE;CN=Bob;PARTSTAT=DECLINED:mailto:bob@example.com" + assert [d.property for d in differences(meeting.icalendar, stored_as(meeting, declined))] == ["ATTENDEE"] + task = Task.new(summary="x") + task.component.add("RELATED-TO", "abc") + child = task.icalendar.to_ical().decode().replace("RELATED-TO:abc", "RELATED-TO;RELTYPE=CHILD:abc") + assert [d.property for d in differences(task.icalendar, icalendar.Calendar.from_ical(child))] == ["RELATED-TO"] + + +def test_an_x_parameter_the_server_adds_is_no_difference() -> None: + sent = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + text = sent.icalendar.to_ical().decode().replace("SUMMARY:x", "SUMMARY;X-SERVER-ID=7:x") + assert differences(sent.icalendar, icalendar.Calendar.from_ical(text)) == [] + + +def test_a_parameter_added_to_one_of_two_lines_is_no_difference() -> None: + sent = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + sent.component.add("ATTENDEE", "mailto:bob@example.com") + sent.component.add("ATTENDEE", "mailto:bob@example.com", parameters={"CN": "Bob"}) + text = sent.icalendar.to_ical().decode() + # The bare line takes the parameter: matched first come first served, the CN=Bob line would be left over. + added = text.replace("ATTENDEE:mailto:", "ATTENDEE;CN=Bob;RSVP=TRUE:mailto:") + assert differences(sent.icalendar, icalendar.Calendar.from_ical(added)) == [] + merged = text.replace("ATTENDEE:mailto:bob@example.com\r\n", "") + assert [d.property for d in differences(sent.icalendar, icalendar.Calendar.from_ical(merged))] == ["ATTENDEE"] def test_an_unescaped_comma_is_a_difference_and_parameter_lists_are_sets() -> None: @@ -142,3 +240,158 @@ def test_failed_on_a_two_state_tracker_is_stored_as_completed() -> None: assert (loss.property, loss.sent, loss.stored) == ("STATUS", "FAILED", "COMPLETED") apply() assert task.component["STATUS"] == "COMPLETED" + + +def test_an_attendee_the_server_added_is_no_difference() -> None: + """Ox adds the organizer as an attendee (RFC 6638 scheduling): that loses nothing.""" + sent = Event.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + sent.component.add("ATTENDEE", "mailto:bob@example.com", parameters={"CN": "Bob"}) + added = stored_as( + sent, + "ATTENDEE;CN=Admin;PARTSTAT=ACCEPTED:mailto:admin@example.com\r\n" + "ATTENDEE;CN=Bob;EMAIL=bob@example.com:mailto:bob@example.com\r\n" + "ORGANIZER;CN=Admin:mailto:admin@example.com", + ) + assert differences(sent.icalendar, added) == [] + stranger = stored_as( + sent, + "ATTENDEE;CN=Eve:mailto:eve@example.com\r\n" + "ATTENDEE;CN=Bob:mailto:bob@example.com\r\n" + "ORGANIZER;CN=Admin:mailto:admin@example.com", + ) + assert "ATTENDEE" in [d.property for d in differences(sent.icalendar, stranger)] + instead = stored_as(sent, "ATTENDEE;CN=Admin:mailto:admin@example.com") + assert [d.property for d in differences(sent.icalendar, instead)] == ["ATTENDEE"] + + +def test_a_dropped_start_is_a_difference() -> None: + """Zimbra drops a task's DTSTART: a loss, not a KeyError.""" + sent = Task.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + stored = copy.deepcopy(sent.icalendar) + del stored.walk("VTODO")[0]["DTSTART"] + [loss] = differences(sent.icalendar, stored) + assert (loss.property, loss.stored) == ("DTSTART", None) + + +def test_a_property_kept_only_beside_another_is_lost_without_it() -> None: + """Zimbra keeps a task's DTSTART only where it has a DUE too.""" + table = {Feature.TASK_START: {"level": Support.LOSSY, "requires": ["DUE"]}} + alone = Task.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC)) + component = copy.deepcopy(alone.component) + [(loss, apply)] = plan_losses(component, Capabilities({**dict.fromkeys(Feature, Support.FULL), **table})) + assert (loss.property, loss.stored) == ("DTSTART", None) + apply() + assert "DTSTART" not in component + both = Task.new(summary="x", start=datetime(2026, 11, 3, 9, tzinfo=UTC), end=datetime(2026, 11, 4, tzinfo=UTC)) + assert plan_losses(both.component, Capabilities({**dict.fromkeys(Feature, Support.FULL), **table})) == [] + + +def test_an_rdate_is_lost_where_only_recurrence_rdate_is_unsupported() -> None: + """Zimbra drops every RDATE and keeps an RRULE.""" + event = Event.new(summary="x", start=datetime(2026, 11, 5, 10, tzinfo=UTC)) + event.component.add("RRULE", {"FREQ": "WEEKLY", "COUNT": 2}) + event.component.add("RDATE", [datetime(2026, 11, 26, 10, tzinfo=UTC)]) + table = {**dict.fromkeys(Feature, Support.FULL), Feature.RECURRENCE_RDATE: Support.UNSUPPORTED} + [(loss, apply)] = plan_losses(event.component, Capabilities(table)) + assert (loss.property, loss.stored) == ("RDATE", None) + apply() + assert "RDATE" not in event.component + assert "RRULE" in event.component + no_recurrence = {**table, Feature.RECURRENCE: Support.UNSUPPORTED} + again = Event.new(summary="x", start=datetime(2026, 11, 5, 10, tzinfo=UTC)) + again.component.add("RDATE", [datetime(2026, 11, 26, 10, tzinfo=UTC)]) + assert [loss.property for loss, _ in plan_losses(again.component, Capabilities(no_recurrence))] == ["RDATE"] + + +def test_a_tasks_rdate_is_kept_where_only_events_lose_theirs() -> None: + """The hint is probed on events: on a calendar holding tasks too, a task's RDATE is not dropped.""" + task = Task.new(summary="x", start=datetime(2026, 11, 5, 10, tzinfo=UTC)) + task.component.add("RDATE", [datetime(2026, 11, 26, 10, tzinfo=UTC)]) + table = {**dict.fromkeys(Feature, Support.FULL), Feature.RECURRENCE_RDATE: Support.UNSUPPORTED} + assert plan_losses(task.component, Capabilities(table)) == [] + + +def test_an_override_of_a_dropped_rdate_goes_with_it() -> None: + """An override of an RDATE occurrence would be an orphan once the RDATE is gone; one the RRULE gives stays.""" + master = Event.new(summary="x", start=datetime(2026, 11, 5, 10, tzinfo=UTC)) + master.component.add("RRULE", {"FREQ": "WEEKLY", "COUNT": 2}) + master.component.add("RDATE", [datetime(2026, 11, 26, 10, tzinfo=UTC)]) + calendar = master.icalendar + for day in (26, 12): # the RDATE's occurrence, and the RRULE's second + override = icalendar.Event() + override.add("UID", master.uid) + override.add("DTSTAMP", datetime(2026, 11, 1, tzinfo=UTC)) + override.add("RECURRENCE-ID", datetime(2026, 11, day, 10, tzinfo=UTC)) + override.add("DTSTART", datetime(2026, 11, day, 11, tzinfo=UTC)) + override.add("SUMMARY", f"moved {day}") + calendar.add_component(override) + table = {**dict.fromkeys(Feature, Support.FULL), Feature.RECURRENCE_RDATE: Support.UNSUPPORTED} + [(loss, apply)] = plan_calendar_losses(calendar, Capabilities(table)) + assert (loss.property, loss.sent, loss.stored) == ("RECURRENCE-ID", "20261126T100000Z", None) + apply() + assert [str(c["SUMMARY"]) for c in calendar.walk("VEVENT") if "RECURRENCE-ID" in c] == ["moved 12"] + + +def overridden(start: date, rrule: dict[str, Any] | None, rdate: Any, end: date | None = None) -> icalendar.Calendar: + """A master with an RDATE and an override of the RDATE's occurrence, which moves it by an hour.""" + master = Event.new(summary="x", start=start) + if end is not None: + master.component.add("DTEND", end) + if rrule is not None: + master.component.add("RRULE", rrule) + master.component.add("RDATE", [rdate]) + calendar = master.icalendar + when = rdate[0] if isinstance(rdate, tuple) else rdate + override = icalendar.Event() + override.add("UID", master.uid) + override.add("DTSTAMP", datetime(2026, 11, 1, tzinfo=UTC)) + override.add("RECURRENCE-ID", when) + override.add("DTSTART", when) + override.add("SUMMARY", "moved") + calendar.add_component(override) + return calendar + + +@pytest.mark.parametrize( + ("calendar", "orphaned"), + [ + pytest.param( + overridden( + datetime(2026, 11, 5, 9, tzinfo=UTC), + {"FREQ": "DAILY", "COUNT": 3}, + datetime(2026, 11, 6, 10, tzinfo=UTC), + end=datetime(2026, 11, 5, 11, tzinfo=UTC), + ), + True, + id="an-rrule-occurrence-overlaps-it", + ), + pytest.param( + overridden( + datetime(2026, 11, 5, 10, tzinfo=UTC), + {"FREQ": "WEEKLY", "COUNT": 2}, + (datetime(2026, 11, 26, 10, tzinfo=UTC), timedelta(hours=1)), + ), + True, + id="period", + ), + pytest.param( + overridden(date(2026, 11, 5), {"FREQ": "WEEKLY", "COUNT": 2}, date(2026, 11, 26)), True, id="all-day" + ), + pytest.param( + overridden(date(2026, 11, 5), {"FREQ": "WEEKLY", "COUNT": 3}, date(2026, 11, 12)), + False, + id="all-day-the-rrule-gives-it-too", + ), + pytest.param( + overridden(datetime(2026, 11, 5, 10, tzinfo=UTC), None, datetime(2026, 11, 26, 10, tzinfo=UTC)), + True, + id="no-rrule", + ), + ], +) +def test_an_rdate_occurrences_override_is_orphaned_unless_the_rrule_starts_one_there( + calendar: icalendar.Calendar, orphaned: bool +) -> None: + table = {**dict.fromkeys(Feature, Support.FULL), Feature.RECURRENCE_RDATE: Support.UNSUPPORTED} + changes = plan_calendar_losses(calendar, Capabilities(table)) + assert [loss.property for loss, _ in changes] == (["RECURRENCE-ID"] if orphaned else []) diff --git a/tests/test_scenarios.py b/tests/test_scenarios.py index 0b38ccd..3a71317 100644 --- a/tests/test_scenarios.py +++ b/tests/test_scenarios.py @@ -24,7 +24,7 @@ from pathlib import Path import pytest -from conftest import caldav_server_names, running_caldav_server, wait_out_search_cache +from conftest import caldav_server_names, running_caldav_server, wait_for_server from icalendar_searcher import Searcher from calendaring import ( @@ -77,7 +77,7 @@ def open_backend(where: str, directory: Path, request: pytest.FixtureRequest) -> return Backend.open(str(directory / "calendar.ics")) server = running_caldav_server(request, where) backend = Backend.open(server.url, kind="caldav", **server.options) - wait_out_search_cache(backend, request.getfixturevalue("monkeypatch")) + wait_for_server(backend, request.getfixturevalue("monkeypatch")) return backend