Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions src/cachier/cores/pickle.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from typing import IO, TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, cast

import portalocker # to lock on pickle cache IO
from watchdog.events import PatternMatchingEventHandler
from watchdog.events import FileSystemEvent, FileSystemEventHandler
from watchdog.observers import Observer

from .._types import HashFunc
Expand All @@ -35,22 +35,31 @@ class _PickleCore(_BaseCore):

_SHARED_LOCK_SUFFIX = ".lock"

class CacheChangeHandler(PatternMatchingEventHandler):
class CacheChangeHandler(FileSystemEventHandler):
"""Handles cache-file modification events."""

def __init__(self, filename, core, key):
PatternMatchingEventHandler.__init__(
self,
patterns=["*" + filename],
ignore_patterns=None,
ignore_directories=True,
case_sensitive=False,
)
FileSystemEventHandler.__init__(self)
self.filename = filename
self.core = core
self.key = key
self.observer = None
self.value = None

def dispatch(self, event: FileSystemEvent) -> None:
"""Dispatch only the events of this cache file.

Matched by file name rather than a glob pattern: watchdog 7 matches
patterns against the whole path (``PurePath.full_match``), so the
former ``"*" + filename`` pattern never matched and waiters hung.

"""
if event.is_directory:
return
paths = [event.src_path, getattr(event, "dest_path", "")]
if any(path and os.path.basename(os.fsdecode(path)) == self.filename for path in paths):
super().dispatch(event)

def inject_observer(self, observer) -> None:
"""Inject the observer running this handler."""
self.observer = observer
Expand Down
31 changes: 31 additions & 0 deletions tests/pickle_tests/test_pickle_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@
from unittest.mock import Mock, patch

import pytest
from watchdog.events import (
DirModifiedEvent,
FileCreatedEvent,
FileModifiedEvent,
FileMovedEvent,
)

try:
import queue
Expand Down Expand Up @@ -265,6 +271,31 @@ def test_pickle_being_calculated(separate_files):
assert res1 == res2


@pytest.mark.pickle
@pytest.mark.parametrize(
("event", "dispatched"),
[
(FileModifiedEvent("/cache/dir/.mod.func"), True),
(FileCreatedEvent("/cache/dir/.mod.func"), True),
(FileMovedEvent("/cache/dir/.mod.func.tmp", "/cache/dir/.mod.func"), True),
(FileModifiedEvent("/cache/dir/.mod.func.lock"), False),
(FileModifiedEvent("/cache/dir/.mod.other_func"), False),
(DirModifiedEvent("/cache/dir/.mod.func"), False),
],
)
def test_pickle_cache_change_handler_matches_the_cache_file_by_name(event, dispatched):
"""Test the waiter's handler reacts to its cache file's events on a full path.

watchdog 7 matches glob patterns against the whole path, which broke a ``"*" + filename`` pattern and left
concurrent waiters hanging.

"""
handler = _PickleCore.CacheChangeHandler(filename=".mod.func", core=Mock(), key="k")
with patch.object(handler, "_check_calculation") as check_calculation:
handler.dispatch(event)
assert check_calculation.called is dispatched


def _being_calc_next_time(arg_1, arg_2):
"""Some function."""
sleep(1)
Expand Down