From d50bf650f39968f7c3c0cf8c9279cef254e4921a Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Sun, 15 Nov 2026 21:27:10 +0000 Subject: [PATCH] stabilize crawler tests that need two requests in parallel --- tests/unit/crawlers/_basic/test_basic_crawler.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index ddde8c4bc3..8d97b5b45d 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -1869,8 +1869,11 @@ async def test_crawler_multiple_stops_in_parallel() -> None: ] processed_urls = [] - # Set concurrency to 2 to ensure two urls are being visited in parallel. - crawler = BasicCrawler(concurrency_settings=ConcurrencySettings(desired_concurrency=2, max_concurrency=2)) + # Set concurrency to 2 to ensure two urls are being visited in parallel. An overloaded machine still runs + # `min_concurrency` tasks. + crawler = BasicCrawler( + concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2) + ) both_handlers_started = asyncio.Barrier(2) only_one_handler_at_a_time = asyncio.Semaphore(1) @@ -2155,7 +2158,8 @@ async def handler(context: BasicCrawlingContext) -> None: async def test_pause_after_abort_on_error_ends_run() -> None: """`pause()` after a request fails with `abort_on_error` waits for the request in progress, and the run ends.""" crawler = BasicCrawler( - concurrency_settings=ConcurrencySettings(desired_concurrency=2, max_concurrency=2), + # An overloaded machine still runs `min_concurrency` tasks, so the second request starts on a busy machine too. + concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2), abort_on_error=True, max_request_retries=0, ) @@ -2478,7 +2482,10 @@ async def test_context_use_state_race_condition_in_handlers(key_value_store: Key Result should be incremented by 2. Method `use_state` must be implemented in a way that prevents race conditions in such scenario.""" - crawler = BasicCrawler() + # An overloaded machine still runs `min_concurrency` tasks, so both handlers run in parallel on a busy machine too. + crawler = BasicCrawler( + concurrency_settings=ConcurrencySettings(min_concurrency=2, desired_concurrency=2, max_concurrency=2) + ) store = await crawler.get_key_value_store() await store.set_value(f'{BasicCrawler._CRAWLEE_STATE_KEY}_0', {'counter': 0}) handler_barrier = asyncio.Barrier(2)