Add py.typed, small improvements (#266)

Author: wRAR

.coveragerc

@@ -9,3 +9,5 @@ docs/_build
 __pycache__/
 /test-results/
 build/
+.idea/
+.vscode/

.gitignore

@@ -68,20 +68,50 @@ parse = "version\\s*=\\s*['\"](?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+
 [[tool.bumpversion.files]]
 filename = "scrapy_zyte_api/__version__.py"
 
+[tool.coverage.run]
+branch = true
+include = [
+    "scrapy_zyte_api/*",
+]
+omit = [
+    "tests/*",
+]
+disable_warnings = [
+    "include-ignored",
+]
+
 [tool.hatch.build.targets.sdist]
 include = [
-  "scrapy_zyte_api/*.py",
+    "/docs",
+    "/scrapy_zyte_api",
+    "/tests",
+    "/CHANGES.rst",
+    "/tox.ini",
 ]
 
 [tool.mypy]
 check_untyped_defs = true
-ignore_missing_imports = true
-no_warn_no_return = true
+implicit_reexport = false
+warn_no_return = false
+warn_unused_ignores = true
 files = [
     "scrapy_zyte_api",
     "tests"
 ]
 
+[[tool.mypy.overrides]]
+module = [
+    "pytest_twisted",
+    "scrapy_crawlera",
+    "scrapy_zyte_smartproxy",
+]
+ignore_missing_imports = true
+
+# to be fixed
+[[tool.mypy.overrides]]
+module = "scrapy_poet.*"
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 junit_family = "xunit2"
 testpaths = [

pyproject.toml

@@ -25,7 +25,7 @@
     from w3lib.url import canonicalize_url
 
     from ._params import _REQUEST_PARAMS, _may_use_browser, _ParamParser
-    from .utils import _build_from_crawler
+    from .utils import _build_from_crawler  # type: ignore[attr-defined]
 
     class ScrapyZyteAPIRequestFingerprinter:
         @classmethod

scrapy_zyte_api/_request_fingerprinter.py

@@ -18,7 +18,7 @@
 from tenacity import stop_after_attempt
 from zyte_api import RequestError, RetryFactory
 
-from .utils import _DOWNLOAD_NEEDS_SPIDER, _build_from_crawler
+from .utils import _DOWNLOAD_NEEDS_SPIDER, _build_from_crawler  # type: ignore[attr-defined]
 
 logger = getLogger(__name__)
 SESSION_INIT_META_KEY = "_is_session_init_request"

scrapy_zyte_api/_session.py

@@ -19,7 +19,7 @@
 
 from ._params import _ParamParser
 from .responses import ZyteAPIResponse, ZyteAPITextResponse, _process_response
-from .utils import (
+from .utils import (  # type: ignore[attr-defined]
     _AUTOTHROTTLE_DONT_ADJUST_DELAY_SUPPORT,
     _X402_SUPPORT,
     USER_AGENT,
@@ -82,7 +82,10 @@ class _ScrapyZyteAPIBaseDownloadHandler:
     lazy = False
 
     def __init__(
-        self, settings: Settings, crawler: Crawler, client: AsyncZyteAPI = None
+        self,
+        settings: Settings,
+        crawler: Crawler,
+        client: Optional[AsyncZyteAPI] = None,
     ):
         if not settings.getbool("ZYTE_API_ENABLED", True):
             raise NotConfigured(
@@ -348,7 +351,10 @@ async def _close(self) -> None:  # NOQA
 
 class ScrapyZyteAPIDownloadHandler(_ScrapyZyteAPIBaseDownloadHandler):
     def __init__(
-        self, settings: Settings, crawler: Crawler, client: AsyncZyteAPI = None
+        self,
+        settings: Settings,
+        crawler: Crawler,
+        client: Optional[AsyncZyteAPI] = None,
     ):
         super().__init__(settings, crawler, client)
         self._fallback_handler = self._create_handler(
@@ -358,7 +364,10 @@ def __init__(
 
 class ScrapyZyteAPIHTTPDownloadHandler(_ScrapyZyteAPIBaseDownloadHandler):
     def __init__(
-        self, settings: Settings, crawler: Crawler, client: AsyncZyteAPI = None
+        self,
+        settings: Settings,
+        crawler: Crawler,
+        client: Optional[AsyncZyteAPI] = None,
     ):
         super().__init__(settings, crawler, client)
         self._fallback_handler = self._create_handler(
@@ -371,7 +380,10 @@ def __init__(
 
 class ScrapyZyteAPIHTTPSDownloadHandler(_ScrapyZyteAPIBaseDownloadHandler):
     def __init__(
-        self, settings: Settings, crawler: Crawler, client: AsyncZyteAPI = None
+        self,
+        settings: Settings,
+        crawler: Crawler,
+        client: Optional[AsyncZyteAPI] = None,
     ):
         super().__init__(settings, crawler, client)
         self._fallback_handler = self._create_handler(

scrapy_zyte_api/handler.py

@@ -368,7 +368,7 @@ async def __call__(  # noqa: C901
             if not kw:
                 continue
             assert issubclass(cls_stripped, Item)
-            result = cls_stripped.from_dict(api_response.raw_api_response[kw])  # type: ignore[attr-defined]
+            result = cls_stripped.from_dict(api_response.raw_api_response[kw])
             if is_typing_annotated(cls):
                 result = AnnotatedInstance(result, cls.__metadata__)  # type: ignore[attr-defined]
             results.append(result)

scrapy_zyte_api/providers.py

@@ -1,3 +1,4 @@
+from __future__ import annotations
 import argparse
 import json
 import socket
@@ -18,7 +19,7 @@
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET, Site
 
-from scrapy_zyte_api._annotations import _ActionResult
+from scrapy_zyte_api._annotations import _ActionResult, ExtractFrom
 from scrapy_zyte_api.responses import _API_RESPONSE
 
 from . import SETTINGS, make_handler
@@ -242,8 +243,6 @@ def render_POST(self, request):
             assert isinstance(response_data["product"]["name"], str)
             extract_from = request_data.get("productOptions", {}).get("extractFrom")
             if extract_from:
-                from scrapy_zyte_api.providers import ExtractFrom
-
                 if extract_from == ExtractFrom.httpResponseBody:
                     response_data["product"]["name"] += " (from httpResponseBody)"
 

scrapy_zyte_api/py.typed

@@ -21,11 +21,11 @@
 from scrapy.settings.default_settings import USER_AGENT as DEFAULT_USER_AGENT
 from scrapy.utils.defer import deferred_f_from_coro_f
 from twisted.internet.defer import Deferred
-from zyte_api.aio.errors import RequestError
+from zyte_api import RequestError
 
 from scrapy_zyte_api._cookies import _get_cookie_jar
-from scrapy_zyte_api._params import ANY_VALUE
-from scrapy_zyte_api.handler import _ParamParser, _ScrapyZyteAPIBaseDownloadHandler
+from scrapy_zyte_api._params import ANY_VALUE, _ParamParser
+from scrapy_zyte_api.handler import _ScrapyZyteAPIBaseDownloadHandler
 from scrapy_zyte_api.responses import _process_response
 from scrapy_zyte_api.utils import _ADDON_SUPPORT