Support upcoming Scrapy changes (#256)

Author: Gallaecio

CHANGES.rst

@@ -1,6 +1,27 @@
 Changes
 =======
 
+Unreleased
+----------
+
+-   Extended :doc:`Scrapy <scrapy:index>` support to :ref:`2.13.0+
+    <scrapy:release-2.13.0>`.
+
+-   Switched the minimum required version of :doc:`python-zyte-api
+    <python-zyte-api:index>` from ``0.5.1`` to ``0.6.0``.
+
+-   Fixed the removal of default request headers (``Accept``,
+    ``Accept-Encoding``, ``Accept-Language``, and ``User-Agent``) not working
+    for request copies (e.g. redirects or retries).
+
+-   The default value of the :setting:`ZYTE_API_FALLBACK_HTTP_HANDLER` and
+    :setting:`ZYTE_API_FALLBACK_HTTPS_HANDLER` settings is as expected even
+    when not using the add-on.
+
+-   The scrapy-zyte-api download handlers now support fallback download
+    handlers that do not define a ``close()`` method.
+
+
 0.29.0 (2025-03-20)
 -------------------
 

docs/reference/settings.rst

@@ -146,6 +146,26 @@ Default: ``False``
 See :ref:`request-automatic`.
 
 
+.. setting:: ZYTE_API_FALLBACK_HTTP_HANDLER
+
+ZYTE_API_FALLBACK_HTTP_HANDLER
+==============================
+
+Default: :setting:`DOWNLOAD_HANDLERS["http"] <scrapy:DOWNLOAD_HANDLERS>`
+
+Download handler to use for HTTP requests that do not go through Zyte API.
+
+
+.. setting:: ZYTE_API_FALLBACK_HTTPS_HANDLER
+
+ZYTE_API_FALLBACK_HTTPS_HANDLER
+===============================
+
+Default: :setting:`DOWNLOAD_HANDLERS["https"] <scrapy:DOWNLOAD_HANDLERS>`
+
+Download handler to use for HTTPS requests that do not go through Zyte API.
+
+
 .. setting:: ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS
 
 ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS

pyproject.toml

@@ -27,7 +27,7 @@ requires-python = ">=3.9"
 dependencies = [
     "packaging>=20.0",
     "scrapy>=2.0.1",
-    "zyte-api>=0.5.1",
+    "zyte-api>=0.6.0",
 ]
 
 [project.optional-dependencies]

scrapy_zyte_api/_middlewares.py

@@ -176,9 +176,19 @@ def __init__(self, crawler):
     def _get_header_set(request):
         return {header.strip().lower() for header in request.headers}
 
-    def process_start_requests(self, start_requests, spider):
+    async def process_start(self, start):
         # Mark start requests and reports to the downloader middleware the
         # number of them once all have been processed.
+        count = 0
+        async for item_or_request in start:
+            if isinstance(item_or_request, Request):
+                count += 1
+                item_or_request.meta["is_start_request"] = True
+                self._process_output_request(item_or_request, None)
+            yield item_or_request
+        self._send_signal(_start_requests_processed, count=count)
+
+    def process_start_requests(self, start_requests, spider):
         count = 0
         for item_or_request in start_requests:
             if isinstance(item_or_request, Request):
@@ -189,7 +199,8 @@ def process_start_requests(self, start_requests, spider):
         self._send_signal(_start_requests_processed, count=count)
 
     def _process_output_request(self, request, spider):
-        request.meta["_pre_mw_headers"] = self._get_header_set(request)
+        if "_pre_mw_headers" not in request.meta:
+            request.meta["_pre_mw_headers"] = self._get_header_set(request)
         self.slot_request(request, spider)
 
     def _process_output_item_or_request(self, item_or_request, spider):

scrapy_zyte_api/handler.py

@@ -323,7 +323,7 @@ def _truncate_params(self, params):
 
     @inlineCallbacks
     def close(self) -> Generator:
-        if self._fallback_handler:
+        if self._fallback_handler and hasattr(self._fallback_handler, "close"):
             yield self._fallback_handler.close()
         yield deferred_from_coro(self._close())
 
@@ -347,7 +347,10 @@ def __init__(
     ):
         super().__init__(settings, crawler, client)
         self._fallback_handler = self._create_handler(
-            settings.get("ZYTE_API_FALLBACK_HTTP_HANDLER")
+            settings.get(
+                "ZYTE_API_FALLBACK_HTTP_HANDLER",
+                settings.getwithbase("DOWNLOAD_HANDLERS")["http"],
+            )
         )
 
 
@@ -357,5 +360,8 @@ def __init__(
     ):
         super().__init__(settings, crawler, client)
         self._fallback_handler = self._create_handler(
-            settings.get("ZYTE_API_FALLBACK_HTTPS_HANDLER")
+            settings.get(
+                "ZYTE_API_FALLBACK_HTTPS_HANDLER",
+                settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
+            )
         )

scrapy_zyte_api/utils.py

@@ -16,7 +16,9 @@
 _SCRAPY_2_4_0 = Version("2.4.0")
 _SCRAPY_2_5_0 = Version("2.5.0")
 _SCRAPY_2_6_0 = Version("2.6.0")
+_SCRAPY_2_10_0 = Version("2.10.0")
 _SCRAPY_2_12_0 = Version("2.12.0")
+_SCRAPY_2_13_0 = Version("2.13.0")
 
 # Need to install an asyncio reactor before download handler imports to work
 # around:
@@ -25,6 +27,8 @@
 # https://github.com/scrapy/scrapy/commit/e4bdd1cb958b7d89b86ea66f0af1cec2d91a6d44
 _NEEDS_EARLY_REACTOR = _SCRAPY_2_4_0 <= _SCRAPY_VERSION < _SCRAPY_2_6_0
 
+_ADDON_SUPPORT = _SCRAPY_VERSION >= _SCRAPY_2_10_0
+_ASYNC_START_SUPPORT = _SCRAPY_VERSION >= _SCRAPY_2_13_0
 _AUTOTHROTTLE_DONT_ADJUST_DELAY_SUPPORT = _SCRAPY_VERSION >= _SCRAPY_2_12_0
 _DOWNLOAD_NEEDS_SPIDER = _SCRAPY_VERSION < _SCRAPY_2_6_0
 _RAW_CLASS_SETTING_SUPPORT = _SCRAPY_VERSION >= _SCRAPY_2_4_0

tests/test_addon.py

@@ -4,6 +4,7 @@
 from pytest_twisted import ensureDeferred
 from scrapy import Request
 from scrapy.core.downloader.handlers.http import HTTP10DownloadHandler
+from scrapy.settings.default_settings import TWISTED_REACTOR
 from scrapy.utils.test import get_crawler
 
 from scrapy_zyte_api import (
@@ -154,11 +155,14 @@ def _test_setting_changes(initial_settings, expected_settings):
         ScrapyZyteAPISpiderMiddleware: 100,
         ScrapyZyteAPIRefererSpiderMiddleware: 1000,
     },
-    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     "ZYTE_API_FALLBACK_HTTPS_HANDLER": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
     "ZYTE_API_FALLBACK_HTTP_HANDLER": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
     "ZYTE_API_TRANSPARENT_MODE": True,
 }
+if TWISTED_REACTOR != "twisted.internet.asyncioreactor.AsyncioSelectorReactor":
+    BASE_EXPECTED["TWISTED_REACTOR"] = (
+        "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+    )
 
 
 @pytest.mark.skipif(