From 405bed4492349354e706dcfaefb656f57f5d184a Mon Sep 17 00:00:00 2001 From: RPBot Date: Wed, 16 Sep 2026 14:51:29 +0000 Subject: [PATCH] Sync web-scraping-with-scrapy-and-mongodb with the maintenance update of "Web Scraping With Scrapy and MongoDB" - Replace the spider's def start_requests(self) with async def start(self); Scrapy removed Spider.start_requests() in 2.16.0, so on current Scrapy the method was never called and the errback on the initial request was silently dropped. - Re-pin requirements.txt to the tutorial's new dependency set (Scrapy 2.18.0, pymongo 4.17.0, Python 3.14). Co-Authored-By: Claude Opus 5 --- .../books/books/spiders/book.py | 2 +- .../requirements.txt | 69 +++++++++---------- 2 files changed, 34 insertions(+), 37 deletions(-) diff --git a/web-scraping-with-scrapy-and-mongodb/books/books/spiders/book.py b/web-scraping-with-scrapy-and-mongodb/books/books/spiders/book.py index a184375532..bcddc7e408 100644 --- a/web-scraping-with-scrapy-and-mongodb/books/books/spiders/book.py +++ b/web-scraping-with-scrapy-and-mongodb/books/books/spiders/book.py @@ -7,7 +7,7 @@ class BookSpider(scrapy.Spider): allowed_domains = ["books.toscrape.com"] start_urls = ["https://books.toscrape.com/"] - def start_requests(self): + async def start(self): for url in self.start_urls: yield scrapy.Request( url, callback=self.parse, errback=self.log_error diff --git a/web-scraping-with-scrapy-and-mongodb/requirements.txt b/web-scraping-with-scrapy-and-mongodb/requirements.txt index bf2b96c05e..ccf47eb600 100644 --- a/web-scraping-with-scrapy-and-mongodb/requirements.txt +++ b/web-scraping-with-scrapy-and-mongodb/requirements.txt @@ -1,40 +1,37 @@ -attrs==23.2.0 -Automat==22.10.0 -certifi==2024.7.4 -cffi==1.16.0 -charset-normalizer==3.3.2 +attrs==26.1.0 +Automat==25.4.16 +brotli==1.2.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 constantly==23.10.4 -cryptography==43.0.0 -cssselect==1.2.0 +cryptography==50.0.1 +cssselect==1.5.0 defusedxml==0.7.1 -dnspython==2.6.1 -filelock==3.15.4 +dnspython==2.8.0 +filelock==3.32.7 hyperlink==21.0.0 -idna==3.7 -incremental==22.10.0 -itemadapter==0.9.0 -itemloaders==1.3.1 -jmespath==1.0.1 -lxml==5.2.2 -packaging==24.1 -parsel==1.9.1 -Protego==0.3.1 -pyasn1==0.6.0 -pyasn1_modules==0.4.0 -pycparser==2.22 +idna==3.19 +incremental==24.11.0 +itemadapter==0.13.1 +itemloaders==1.4.0 +jmespath==1.1.0 +lxml==6.1.3 +packaging==26.3 +parsel==1.11.0 +Protego==0.6.2 +pycparser==3.0 PyDispatcher==2.0.7 -pymongo==4.8.0 -pyOpenSSL==24.2.1 -queuelib==1.7.0 -requests==2.32.3 -requests-file==2.1.0 -Scrapy==2.11.2 -service-identity==24.1.0 -setuptools==71.1.0 -six==1.16.0 -tldextract==5.1.2 -Twisted==24.3.0 -typing_extensions==4.12.2 -urllib3==2.2.2 -w3lib==2.2.1 -zope.interface==6.4.post2 +pymongo==4.17.0 +pyOpenSSL==26.4.0 +queuelib==1.10.0 +requests==2.34.2 +requests-file==3.0.1 +Scrapy==2.18.0 +service-identity==26.1.0 +tldextract==5.3.2 +Twisted==26.4.0 +typing_extensions==4.16.0 +urllib3==2.8.0 +w3lib==2.4.1 +zope.interface==8.6