Merge pull request #260 from scrapy-plugins/poetless

Do not use poet for request fingerprinting unless configured

Author: kmike

scrapy_zyte_api/_request_fingerprinter.py

@@ -32,32 +32,44 @@ class ScrapyZyteAPIRequestFingerprinter:
         def from_crawler(cls, crawler):
             return cls(crawler)
 
-        def __init__(self, crawler):
-            settings = crawler.settings
+        @staticmethod
+        def _poet_is_configured(settings):
             try:
-                from scrapy_poet import ScrapyPoetRequestFingerprinter
+                from scrapy_poet import InjectionMiddleware
             except ImportError:
-                self._has_poet = False
-                RequestFingerprinter = ScrapyRequestFingerprinter
+                return False
+            for k, v in settings.get("DOWNLOADER_MIDDLEWARES", {}).items():
+                if issubclass(load_object(k), InjectionMiddleware):
+                    return v is not None
+            return False
+
+        def __init__(self, crawler):
+            settings = crawler.settings
+            self._fallback_fingerprinter_is_poets = poet_is_configured = (
+                self._poet_is_configured(settings)
+            )
+            if poet_is_configured:
+                from scrapy_poet import (
+                    ScrapyPoetRequestFingerprinter as DefaultFallbackRequestFingerprinter,
+                )
             else:
-                self._has_poet = True
-                RequestFingerprinter = ScrapyPoetRequestFingerprinter
+                DefaultFallbackRequestFingerprinter = ScrapyRequestFingerprinter
             self._fallback_request_fingerprinter = _build_from_crawler(
                 load_object(
                     settings.get(
                         "ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS",
-                        RequestFingerprinter,
+                        DefaultFallbackRequestFingerprinter,
                     )
                 ),
                 crawler,
             )
-            if self._has_poet and not isinstance(
-                self._fallback_request_fingerprinter, cast(type, RequestFingerprinter)
+            if poet_is_configured and not isinstance(
+                self._fallback_request_fingerprinter,
+                cast(type, DefaultFallbackRequestFingerprinter),
             ):
                 logger.warning(
-                    f"You have scrapy-poet installed, but your custom value "
-                    f"for the ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS "
-                    f"setting "
+                    f"scrapy-poet is enabled, but your custom value for the "
+                    f"ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS setting "
                     f"({settings['ZYTE_API_FALLBACK_REQUEST_FINGERPRINTER_CLASS']!r})"
                     f" does not point to a subclass of "
                     f"scrapy_poet.ScrapyPoetRequestFingerprinter. For request "
@@ -69,7 +81,7 @@ def __init__(self, crawler):
                     f"to the SCRAPY_POET_REQUEST_FINGERPRINTER_BASE_CLASS "
                     f"setting instead."
                 )
-                self._has_poet = False
+                self._fallback_fingerprinter_is_poets = False
             self._cache: "WeakKeyDictionary[Request, bytes]" = WeakKeyDictionary()
             self._param_parser = _ParamParser(crawler, cookies_enabled=False)
             self._crawler = crawler
@@ -124,7 +136,7 @@ def fingerprint(self, request):
                     api_params.setdefault("sessionContext", session_pool)
                 self._normalize_params(api_params)
                 fingerprint = json.dumps(api_params, sort_keys=True).encode()
-                if self._has_poet:
+                if self._fallback_fingerprinter_is_poets:
                     deps_key = self._fallback_request_fingerprinter.get_deps_key(
                         request
                     )

tests/test_request_fingerprinter.py

@@ -57,17 +57,42 @@ def fingerprint(self, request):
     assert fingerprinter.fingerprint(request) == b"foo"
     request = Request("https://example.com", meta={"zyte_api": True})
     assert fingerprinter.fingerprint(request) != b"foo"
-    try:
-        import scrapy_poet  # noqa: F401
-    except ImportError:
-        pass
-    else:
+    if scrapy_poet is not None:
         assert (
             "does not point to a subclass of scrapy_poet.ScrapyPoetRequestFingerprinter"
             in caplog.text
         )
 
 
+@pytest.mark.skipif(scrapy_poet is None, reason="scrapy-poet is not installed")
+@ensureDeferred
+async def test_poet_installed_but_disabled(caplog):
+    """If the scrapy-poet package is installed but its main middleware,
+    InjectionMiddleware, is not set in DOWNLOADER_MIDDLEWARES, do not try to
+    use its API for request fingerprinting."""
+    from web_poet import WebPage
+
+    no_deps_request = Request("https://example.com")
+
+    class DepsSpider(Spider):
+        name = "deps"
+
+        def __init__(self, *args, **kwargs):
+            self.deps_request = Request("https://example.com", callback=self.parse_deps)
+
+        def parse_deps(self, response, a: WebPage):
+            pass
+
+    crawler = await get_crawler(
+        spider_cls=DepsSpider, settings={"ZYTE_API_TRANSPARENT_MODE": True}, poet=False
+    )
+    fingerprinter = crawler.request_fingerprinter
+
+    no_deps_fp = fingerprinter.fingerprint(no_deps_request)
+    deps_fp = fingerprinter.fingerprint(crawler.spider.deps_request)
+    assert no_deps_fp == deps_fp
+
+
 @ensureDeferred
 async def test_fallback_default():
     crawler = await get_crawler()