diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c5a3f8ba..41ddfdc1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,12 +8,13 @@ updates: - package-ecosystem: pip directory: / schedule: - interval: monthly + interval: quarterly ignore: - - dependency-name: ruff - dependency-name: bandit + - dependency-name: ruff + - dependency-name: ssort - package-ecosystem: github-actions directory: / schedule: - interval: monthly + interval: quarterly diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 22153aee..fb7b4681 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -13,7 +13,7 @@ on: permissions: read-all jobs: - test: + pytest: name: Pytest testing runs-on: ${{ matrix.os }} @@ -21,15 +21,18 @@ jobs: fail-fast: false matrix: python-version: - - '3.9' - '3.10' - '3.11' - '3.12' - '3.13' + - '3.14' os: - ubuntu-latest - windows-latest - macos-latest + resolution: + - highest + - lowest-direct permissions: contents: write @@ -47,27 +50,26 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - - run: uv pip install --system -e .[dev] + - run: uv pip install --system --resolution ${{ matrix.resolution }} -e .[dev] - id: cache-pytest uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: .pytest_cache - key: ${{ runner.os }}-pytest-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} + key: ${{ matrix.os }}-pytest-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }} - - name: Run pytest (with headless support) - uses: GabrielBB/xvfb-action@5bcda06da84ba084708898801da79736b88e00a9 + - uses: GabrielBB/xvfb-action@5bcda06da84ba084708898801da79736b88e00a9 env: - COVERAGE_FILE: .coverage.${{ runner.os }}.${{ matrix.python-version }} + COVERAGE_FILE: .coverage.${{ matrix.os }}.${{ matrix.python-version }}.${{ matrix.resolution }} with: run: pytest - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 with: - name: coverage-${{ runner.os }}${{ matrix.python-version }} - path: .coverage.${{ runner.os }}.${{ matrix.python-version }} + name: coverage-${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.resolution }} + path: .coverage.${{ matrix.os }}.${{ matrix.python-version }}.${{ matrix.resolution }} include-hidden-files: true ruff-format: @@ -92,7 +94,7 @@ jobs: python-version: '3.13' cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - run: uv pip install --system -e .[dev] @@ -131,7 +133,7 @@ jobs: python-version: '3.13' cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - run: uv pip install --system -e .[dev] @@ -145,7 +147,7 @@ jobs: run: | ruff check --output-format=sarif -o results.sarif . - - uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee + - uses: github/codeql-action/upload-sarif@e12f0178983d466f2f6028f5cc7a6d786fd97f4b if: ( success() || failure() ) && contains('["success", "failure"]', steps.run-ruff-sarif.outcome) with: sarif_file: results.sarif @@ -178,7 +180,7 @@ jobs: python-version: '3.13' cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - run: uv pip install --system -e .[dev] @@ -218,7 +220,7 @@ jobs: python-version: '3.13' cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - run: uv pip install --system -e .[dev] @@ -226,7 +228,7 @@ jobs: run: | bandit --confidence-level 'medium' --format 'sarif' --output 'results.sarif' --recursive 'requestium' - - uses: github/codeql-action/upload-sarif@0499de31b99561a6d14a36a5f662c2a54f91beee + - uses: github/codeql-action/upload-sarif@e12f0178983d466f2f6028f5cc7a6d786fd97f4b if: ( success() || failure() ) && contains('["success", "failure"]', steps.run-bandit-sarif.outcome) with: sarif_file: results.sarif @@ -238,7 +240,7 @@ jobs: coverage: runs-on: ubuntu-latest - needs: test + needs: pytest permissions: pull-requests: write contents: write @@ -254,7 +256,7 @@ jobs: - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 with: pattern: coverage-* merge-multiple: true @@ -300,7 +302,7 @@ jobs: python-version: '3.13' cache: pip - - uses: install-pinned/uv@3863536aec631cbd0a0d99cc91d32d06292bcb93 + - uses: install-pinned/uv@3b52ff50f07de12f5ebcdd31d078466611a008a6 - run: uv pip install --system -e .[dev] @@ -308,7 +310,7 @@ jobs: uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: .pre-commit-cache - key: ${{ runner.os }}-pre-commit-3.13 + key: ${{ runner.os }}-pre-commit-3.13-${{ hashFiles('.pre-commit-config.yaml') }} - name: Run pre-commit on all files run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cfc3703d..ddd6a539 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 'v5.0.0' + rev: 'v6.0.0' hooks: - id: check-yaml - id: check-ast @@ -13,19 +13,19 @@ repos: - id: check-toml - id: debug-statements - id: mixed-line-ending - - repo: https://github.com/asottile/pyupgrade - rev: 'v3.19.1' + - repo: https://github.com/bwhmather/ssort + rev: 0.16.0 hooks: - - id: pyupgrade - args: ['--py39-plus'] + - id: ssort - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 'v0.11.11' + rev: 'v0.14.11' hooks: - - id: ruff + - id: ruff-check + args: ['--fix'] - id: ruff-format - repo: https://github.com/PyCQA/bandit - rev: '1.8.3' + rev: '1.9.2' hooks: - id: bandit args: ['--confidence-level', 'medium'] - files: '^requestium' \ No newline at end of file + files: '^src' diff --git a/pyproject.toml b/pyproject.toml index 4654e3b0..f9c08290 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,24 @@ [project] name = "requestium" -version = "0.5.0" +version = "1.0.0" readme = { file = "README.md", content-type = "text/markdown" } -requires-python = ">=3.9" +requires-python = ">=3.10" license = { file = "LICENSE" } authors = [ { name = "Joaquin Alori", email = "joaquin@tryolabs.com" } ] maintainers = [ { name = "Judson Neer", email = "jkudson.neer@gmail.com" }, - { name = "Wil T", email = "wil.t.me@pm.me" }, -] -dependencies = [ - "parsel>=1.0", - "requests>=2.0", - "selenium>=4.0", - "tldextract>=5.3", ] classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Natural Language :: English", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", @@ -36,6 +29,12 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Testing", ] +dependencies = [ + "parsel>=1.7", + "requests>=2.27", + "selenium>=4.32", + "tldextract>=5.3", +] [project.urls] source = "https://github.com/tryolabs/requestium" @@ -44,20 +43,21 @@ issues = "https://github.com/tryolabs/requestium/issues" [project.optional-dependencies] dev = [ - "bandit[sarif]==1.8.3", - "coverage==7.10.7", + "bandit[sarif]==1.9.2", "mypy==1.19.1", - "pre-commit==4.3.0", - "pytest-cov==7.0.0", - "pytest-xdist==3.8.0", - "pytest==8.4.2", - "ruff==0.11.11", + "pre-commit~=4.5.0", + "pytest-cov~=7.0.0", + "pytest-xdist~=3.8.0", + "pytest~=9.0.0", + "ruff==0.14.11", + "ssort==0.16.0", "types-requests==2.32.4.20250913", ] +[tool] + [tool.ruff] line-length = 160 -target-version = "py39" include = [ "requestium/**/*.py", "tests/**/*.py", @@ -90,6 +90,7 @@ extend-select = [ "LOG", # flake8-logging (LOG) "N", # pep8-naming (N) "PIE", # flake8-pie (PIE) + "PL", # Pylint (PL) "PT", # flake8-pytest-style (PT) "PTH", # flake8-use-pathlib (PTH) "Q", # flake8-quotes (Q) @@ -121,28 +122,24 @@ ignore = [ ] [tool.ruff.lint.per-file-ignores] -"**/{tests,docs}/*" = ["SLF001"] +"**/{tests,docs}/*" = [ + "SLF001", # private-member-access (SLF001) + "PLR2004", # magic-value-comparison (PLR2004) +] -[tool.pytest.ini_options] -addopts = "--cov=requestium -n auto" +[tool.pytest] +addopts = ["-n", "auto", "--cov=requestium", "--no-cov-on-fail"] testpaths = [ - "tests/", + "tests", ] +[tool.coverage] + [tool.coverage.run] branch = true relative_files = true -command_line = "-m pytest" - -[tool.coverage.paths] -source = [ - "requestium/", -] -omit = [ - "tests/", -] [tool.coverage.report] exclude_also = [ - "logger.", + "logger\\.", ] diff --git a/requestium/requestium_mixin.py b/requestium/requestium_mixin.py index 112fb3a8..0483a84d 100644 --- a/requestium/requestium_mixin.py +++ b/requestium/requestium_mixin.py @@ -20,6 +20,45 @@ DEFAULT_TIMEOUT: float = 0.5 +def _ensure_click(self: WebElement) -> None: + """ + Ensure a click gets made, because Selenium can be a bit buggy about clicks. + + This method gets added to the selenium element returned in '__ensure_element_by_xpath'. + We should probably add it to more selenium methods, such as all the 'find**' methods though. + + I wrote this method out of frustration with chromedriver and its problems with clicking + items that need to be scrolled to in order to be clickable. In '__ensure_element_by_xpath' we + scroll to the item before returning it, but chrome has some problems if it doesn't get some + time to scroll to the item. This method ensures chromes gets enough time to scroll to the item + before clicking it. I tried SEVERAL more 'correct' methods to get around this, but none of them + worked 100% of the time (waiting for the element to be 'clickable' does not work). + """ + # We ensure the element is scrolled into the middle of the viewport to ensure that + # it is clickable. There are two main ways an element may not be clickable: + # - It is outside of the viewport + # - It is under a banner or toolbar + # This script solves both cases + script = ( + "var viewPortHeight = Math.max(" + "document.documentElement.clientHeight, window.innerHeight || 0);" + "var elementTop = arguments[0].getBoundingClientRect().top;" + "window.scrollBy(0, elementTop-(viewPortHeight/2));" + ) + self.parent.execute_script(script, self) # parent = the webdriver + + exception_message = "" + for _ in range(10): + try: + self.click() + return + except WebDriverException as e: + exception_message = str(e) + time.sleep(0.2) + msg = f"Couldn't click item after trying 10 times, got error message: \n{exception_message}" + raise WebDriverException(msg) + + class DriverMixin(RemoteWebDriver): """Provides helper methods to our driver classes.""" @@ -214,42 +253,3 @@ def re(self, *args, **kwargs) -> list[str]: def re_first(self, *args, **kwargs) -> str | None: return self.selector.re_first(*args, **kwargs) - - -def _ensure_click(self: WebElement) -> None: - """ - Ensure a click gets made, because Selenium can be a bit buggy about clicks. - - This method gets added to the selenium element returned in '__ensure_element_by_xpath'. - We should probably add it to more selenium methods, such as all the 'find**' methods though. - - I wrote this method out of frustration with chromedriver and its problems with clicking - items that need to be scrolled to in order to be clickable. In '__ensure_element_by_xpath' we - scroll to the item before returning it, but chrome has some problems if it doesn't get some - time to scroll to the item. This method ensures chromes gets enough time to scroll to the item - before clicking it. I tried SEVERAL more 'correct' methods to get around this, but none of them - worked 100% of the time (waiting for the element to be 'clickable' does not work). - """ - # We ensure the element is scrolled into the middle of the viewport to ensure that - # it is clickable. There are two main ways an element may not be clickable: - # - It is outside of the viewport - # - It is under a banner or toolbar - # This script solves both cases - script = ( - "var viewPortHeight = Math.max(" - "document.documentElement.clientHeight, window.innerHeight || 0);" - "var elementTop = arguments[0].getBoundingClientRect().top;" - "window.scrollBy(0, elementTop-(viewPortHeight/2));" - ) - self.parent.execute_script(script, self) # parent = the webdriver - - exception_message = "" - for _ in range(10): - try: - self.click() - return - except WebDriverException as e: - exception_message = str(e) - time.sleep(0.2) - msg = f"Couldn't click item after trying 10 times, got error message: \n{exception_message}" - raise WebDriverException(msg) diff --git a/requestium/requestium_session.py b/requestium/requestium_session.py index 28f710c2..59acc7db 100644 --- a/requestium/requestium_session.py +++ b/requestium/requestium_session.py @@ -30,43 +30,6 @@ class Session(requests.Session): Some useful helper methods and object wrappings have been added. """ - def __init__( - self, - webdriver_path: str | None = None, - headless: bool | None = None, - default_timeout: float = 5, - webdriver_options: dict[str, Any] | None = None, - driver: DriverMixin | None = None, - ) -> None: - super().__init__() - - if webdriver_options is None: - webdriver_options = {} - - self.webdriver_path = webdriver_path - self.default_timeout = default_timeout - self.webdriver_options = webdriver_options - self._driver = driver - self._last_requests_url: str | None = None - - if not self._driver: - self._driver_initializer = functools.partial(self._start_chrome_browser, headless=headless) - else: - for name in DriverMixin.__dict__: - name_private = name.startswith("__") and name.endswith("__") - name_function = isinstance(DriverMixin.__dict__[name], types.FunctionType) - name_in_driver = name in dir(self._driver) - if name_private or not name_function or name_in_driver: - continue - self._driver.__dict__[name] = DriverMixin.__dict__[name].__get__(self._driver) - self._driver.default_timeout = self.default_timeout - - @property - def driver(self) -> DriverMixin: - if self._driver is None: - self._driver = self._driver_initializer() - return self._driver - def _start_chrome_browser(self, headless: bool | None = False): # noqa C901 # TODO @joaqo: Transfer of proxies and headers. # https://github.com/tryolabs/requestium/issues/96 @@ -109,6 +72,44 @@ def _start_chrome_browser(self, headless: bool | None = False): # noqa C901 service = ChromeService(executable_path=self.webdriver_path) return RequestiumChrome(service=service, options=chrome_options, default_timeout=self.default_timeout) + def __init__( + self, + *, + webdriver_path: str | None = None, + headless: bool | None = None, + default_timeout: float = 5, + webdriver_options: dict[str, Any] | None = None, + driver: DriverMixin | None = None, + ) -> None: + super().__init__() + + if webdriver_options is None: + webdriver_options = {} + + self.webdriver_path = webdriver_path + self.default_timeout = default_timeout + self.webdriver_options = webdriver_options + self._driver = driver + self._last_requests_url: str | None = None + + if not self._driver: + self._driver_initializer = functools.partial(self._start_chrome_browser, headless=headless) + else: + for name in DriverMixin.__dict__: + name_private = name.startswith("__") and name.endswith("__") + name_function = isinstance(DriverMixin.__dict__[name], types.FunctionType) + name_in_driver = name in dir(self._driver) + if name_private or not name_function or name_in_driver: + continue + self._driver.__dict__[name] = DriverMixin.__dict__[name].__get__(self._driver) + self._driver.default_timeout = self.default_timeout + + @property + def driver(self) -> DriverMixin: + if self._driver is None: + self._driver = self._driver_initializer() + return self._driver + def transfer_session_cookies_to_driver(self, domain: str | None = None) -> None: """ Copy the Session's cookies into the webdriver. diff --git a/tests/conftest.py b/tests/conftest.py index b8c455ae..cfc66f9a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -77,11 +77,15 @@ def session(request: FixtureRequest) -> Generator[requestium.Session, None, None msg = f"Unknown driver type: {browser}" raise ValueError(msg) - session = requestium.Session(driver=cast("DriverMixin", driver)) - yield session - - with contextlib.suppress(WebDriverException, OSError): - driver.quit() + try: + assert driver.name in browser + session = requestium.Session(driver=cast("DriverMixin", driver)) + assert session.driver.name in browser + + yield session + finally: + with contextlib.suppress(WebDriverException, OSError): + driver.quit() @pytest.fixture(autouse=True)