Merge pull request #261 from scrapy/improve-typing-1

Fix mypy issues in parsel.

Author: wRAR

parsel/selector.py

@@ -4,16 +4,40 @@
 
 import typing
 import warnings
-from typing import Any, Dict, List, Mapping, Optional, Pattern, Union
+from typing import (
+    Any,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Pattern,
+    Type,
+    TypeVar,
+    Union,
+)
 from warnings import warn
 
+from cssselect import GenericTranslator as OriginalGenericTranslator
 from lxml import etree, html
 from pkg_resources import parse_version
 
 from .csstranslator import GenericTranslator, HTMLTranslator
 from .utils import extract_regex, flatten, iflatten, shorten
 
-_SelectorType = typing.TypeVar("_SelectorType", bound="Selector")
+
+if typing.TYPE_CHECKING:
+    # both require Python 3.8
+    from typing import Literal, SupportsIndex
+
+    # simplified _OutputMethodArg from types-lxml
+    _TostringMethodType = Literal[
+        "html",
+        "xml",
+    ]
+
+
+_SelectorType = TypeVar("_SelectorType", bound="Selector")
+_ParserType = Union[etree.XMLParser, etree.HTMLParser]
 
 lxml_version = parse_version(etree.__version__)
 lxml_huge_tree_version = parse_version("4.2")
@@ -62,16 +86,20 @@ def _st(st: Optional[str]) -> str:
 
 
 def create_root_node(
-    text, parser_cls, base_url=None, huge_tree=LXML_SUPPORTS_HUGE_TREE
-):
+    text: str,
+    parser_cls: Type[_ParserType],
+    base_url: Optional[str] = None,
+    huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
+) -> etree._Element:
     """Create root node for text using given parser class."""
     body = text.strip().replace("\x00", "").encode("utf8") or b"<html/>"
     if huge_tree and LXML_SUPPORTS_HUGE_TREE:
         parser = parser_cls(recover=True, encoding="utf8", huge_tree=True)
-        root = etree.fromstring(body, parser=parser, base_url=base_url)
+        # the stub wrongly thinks base_url can't be None
+        root = etree.fromstring(body, parser=parser, base_url=base_url)  # type: ignore[arg-type]
     else:
         parser = parser_cls(recover=True, encoding="utf8")
-        root = etree.fromstring(body, parser=parser, base_url=base_url)
+        root = etree.fromstring(body, parser=parser, base_url=base_url)  # type: ignore[arg-type]
         for error in parser.error_log:
             if "use XML_PARSE_HUGE option" in error.message:
                 warnings.warn(
@@ -90,18 +118,23 @@ class SelectorList(List[_SelectorType]):
     """
 
     @typing.overload
-    def __getitem__(self, pos: int) -> _SelectorType:
+    def __getitem__(self, pos: "SupportsIndex") -> _SelectorType:
         pass
 
     @typing.overload
     def __getitem__(self, pos: slice) -> "SelectorList[_SelectorType]":
         pass
 
     def __getitem__(
-        self, pos: Union[int, slice]
+        self, pos: Union["SupportsIndex", slice]
     ) -> Union[_SelectorType, "SelectorList[_SelectorType]"]:
         o = super().__getitem__(pos)
-        return self.__class__(o) if isinstance(pos, slice) else o
+        if isinstance(pos, slice):
+            return self.__class__(
+                typing.cast("SelectorList[_SelectorType]", o)
+            )
+        else:
+            return typing.cast(_SelectorType, o)
 
     def __getstate__(self) -> None:
         raise TypeError("can't pickle SelectorList objects")
@@ -237,7 +270,7 @@ def attrib(self) -> Mapping[str, str]:
             return x.attrib
         return {}
 
-    def remove(self) -> None:
+    def remove(self) -> None:  # type: ignore[override]
         """
         Remove matched nodes from the parent for each element in this list.
         """
@@ -308,9 +341,15 @@ def __init__(
         huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
     ) -> None:
         self.type = st = _st(type or self._default_type)
-        self._parser = _ctgroup[st]["_parser"]
-        self._csstranslator = _ctgroup[st]["_csstranslator"]
-        self._tostring_method = _ctgroup[st]["_tostring_method"]
+        self._parser: Type[_ParserType] = typing.cast(
+            Type[_ParserType], _ctgroup[st]["_parser"]
+        )
+        self._csstranslator: OriginalGenericTranslator = typing.cast(
+            OriginalGenericTranslator, _ctgroup[st]["_csstranslator"]
+        )
+        self._tostring_method: "_TostringMethodType" = typing.cast(
+            "_TostringMethodType", _ctgroup[st]["_tostring_method"]
+        )
 
         if text is not None:
             if not isinstance(text, str):
@@ -334,7 +373,7 @@ def _get_root(
         text: str,
         base_url: Optional[str] = None,
         huge_tree: bool = LXML_SUPPORTS_HUGE_TREE,
-    ) -> Any:
+    ) -> etree._Element:
         return create_root_node(
             text, self._parser, base_url=base_url, huge_tree=huge_tree
         )
@@ -365,7 +404,9 @@ def xpath(
         try:
             xpathev = self.root.xpath
         except AttributeError:
-            return self.selectorlist_cls([])
+            return typing.cast(
+                SelectorList[_SelectorType], self.selectorlist_cls([])
+            )
 
         nsp = dict(self.namespaces)
         if namespaces is not None:
@@ -389,7 +430,9 @@ def xpath(
             )
             for x in result
         ]
-        return self.selectorlist_cls(result)
+        return typing.cast(
+            SelectorList[_SelectorType], self.selectorlist_cls(result)
+        )
 
     def css(self: _SelectorType, query: str) -> SelectorList[_SelectorType]:
         """
@@ -404,7 +447,7 @@ def css(self: _SelectorType, query: str) -> SelectorList[_SelectorType]:
         """
         return self.xpath(self._css2xpath(query))
 
-    def _css2xpath(self, query: str) -> Any:
+    def _css2xpath(self, query: str) -> str:
         return self._csstranslator.css_to_xpath(query)
 
     def re(
@@ -512,7 +555,10 @@ def remove_namespaces(self) -> None:
             # loop on element attributes also
             for an in el.attrib:
                 if an.startswith("{"):
-                    el.attrib[an.split("}", 1)[1]] = el.attrib.pop(an)
+                    # this cast shouldn't be needed as pop never returns None
+                    el.attrib[an.split("}", 1)[1]] = typing.cast(
+                        str, el.attrib.pop(an)
+                    )
         # remove namespace declarations
         etree.cleanup_namespaces(self.root)
 
@@ -537,7 +583,7 @@ def remove(self) -> None:
             )
 
         try:
-            parent.remove(self.root)
+            parent.remove(self.root)  # type: ignore[union-attr]
         except AttributeError:
             # 'NoneType' object has no attribute 'remove'
             raise CannotRemoveElementWithoutParent(

parsel/utils.py

@@ -1,5 +1,5 @@
 import re
-from typing import Any, List, Pattern, Union
+from typing import Any, List, Pattern, Union, cast, Match
 from w3lib.html import replace_entities as w3lib_replace_entities
 
 
@@ -69,7 +69,7 @@ def extract_regex(
     if "extract" in regex.groupindex:
         # named group
         try:
-            extracted = regex.search(text).group("extract")
+            extracted = cast(Match[str], regex.search(text)).group("extract")
         except AttributeError:
             strings = []
         else:

tests/test_selector.py

@@ -3,9 +3,12 @@
 import weakref
 import unittest
 import pickle
+
+import typing
 from typing import Any
 
 from lxml import etree
+from lxml.html import HtmlElement
 from pkg_resources import parse_version
 
 from parsel import Selector, SelectorList
@@ -718,7 +721,7 @@ def test_namespaces_multiple_adhoc(self) -> None:
     def test_make_links_absolute(self) -> None:
         text = '<a href="file.html">link to file</a>'
         sel = Selector(text=text, base_url="http://example.com")
-        sel.root.make_links_absolute()
+        typing.cast(HtmlElement, sel.root).make_links_absolute()
         self.assertEqual(
             "http://example.com/file.html",
             sel.xpath("//a/@href").extract_first(),

tox.ini

@@ -23,10 +23,12 @@ commands =
 [testenv:typing]
 deps =
     {[testenv]deps}
+    types-lxml==2022.4.10
+    types-psutil==5.9.5.4
     types-setuptools==65.5.0.1
     mypy==0.982
 commands =
-    mypy {posargs:tests} --warn-unused-ignores --ignore-missing-imports
+    mypy {posargs:parsel tests} --warn-unused-ignores
 
 [testenv:pylint]
 deps =