From d810d091617a984153c7779a899843cf1ed1bc35 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:51:44 +0200 Subject: [PATCH 1/2] Support more typing constructs and make Hashable and Sized value types --- CHANGELOG.rst | 23 +++ DOCUMENTATION.rst | 24 +-- jsonargparse/_completions_jsonschema.py | 11 +- jsonargparse/_namespace.py | 2 +- jsonargparse/_parameter_resolvers.py | 13 +- jsonargparse/_postponed_annotations.py | 6 +- jsonargparse/_typehints.py | 72 ++++++-- jsonargparse_tests/test_attrs.py | 15 +- .../test_completions_jsonschema.py | 23 +++ jsonargparse_tests/test_dataclasses.py | 28 ++- jsonargparse_tests/test_subclasses.py | 8 + jsonargparse_tests/test_typehints.py | 162 +++++++++++++++++- 12 files changed, 352 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 45eb3a69..fa29ab44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -61,6 +61,10 @@ Added `__). - New ``json_or_yaml`` parser mode, which parses as JSON and if that fails as YAML (`#1004 `__). +- Support for ``Counter``, ``ChainMap`` and ``Iterator`` (a list until + instantiation) types, and for fields of dataclass-like types typed + ``Final[X]`` or ``InitVar[X]``, parsed as ``X``. Previously these were not + validated (`#1007 `__). Fixed ^^^^^ @@ -154,6 +158,20 @@ Fixed - ``jsonnet`` parser mode hiding Jsonnet evaluation errors when the config also parses as YAML, e.g. ``{"a": error "boom"}`` giving ``a`` the string ``error "boom"`` (`#1004 `__). +- Variadic tuples, e.g. ``tuple[int, *Ts]`` or ``tuple[int, *tuple[str, ...]]``, + and ``*args: *Ts`` failing with ``The code should never reach here`` (python + 3.10: ``Unable to add parameter``) or validated as having a fixed number of + elements. Now they are unvalidated (`#1007 + `__). +- Signature parameters with an unhashable type, e.g. ``P.args`` of a + ``ParamSpec``, failing with ``unhashable type`` instead of being unvalidated + (`#1007 `__). +- A subclass spec with ``init_args`` whose ``class_path`` is the import path of + an instance of the type giving that instance and silently ignoring the + ``init_args`` (`#1007 + `__). +- ``dict`` arguments with a ``Counter`` default parsed as an empty ``Counter`` + (`#1007 `__). Changed ^^^^^^^ @@ -238,6 +256,11 @@ Changed without the ``yaml`` extra makes YAML configs fail. Load errors now say which format failed to parse (`#1004 `__). +- ``Hashable`` and ``Sized`` are now value types instead of subclass types: a + value is accepted if it is hashable or has a length, the config schema + describes the accepted values, a list given for ``Hashable`` becomes a tuple, + and ``Sized`` rejects subclass specs (`#1007 + `__). Removed ^^^^^^^ diff --git a/DOCUMENTATION.rst b/DOCUMENTATION.rst index e817c14a..f21bb34e 100644 --- a/DOCUMENTATION.rst +++ b/DOCUMENTATION.rst @@ -453,18 +453,20 @@ Types can be nested with any complexity. Notes about the support: - Fully supported types are: ``str``/``LiteralString``, ``bool`` (see :ref:`boolean-arguments`), ``int``, ``float``, ``Decimal``, ``complex``, ``bytes``/``bytearray`` (Base64 encoding), ``range``, ``list`` (see - :ref:`list-append`), ``Deque``, ``Iterable``, ``Sequence``, - ``MutableSequence``, ``Collection``, ``Container``, ``Reversible``, - ``Any``/``object``, ``Union``/``Optional`` (see :ref:`union-types`), - ``Literal``, ``Type``, ``Enum``, ``PathLike``, ``UUID``, ``Fraction``, - ``re.Pattern`` (str only), ``datetime``/``date``/``time`` (ISO 8601), - ``timedelta``, the restricted types of :ref:`restricted-numbers` and + :ref:`list-append`), ``Deque``, ``Iterable``, ``Iterator`` (a list until + instantiation), ``Sequence``, ``MutableSequence``, ``Collection``, + ``Container``, ``Reversible``, ``Hashable`` (lists become tuples), + ``Sized`` (no subclass specs, for an object use a link, see + :ref:`argument-linking`), ``Any``/``object``, ``Union``/``Optional`` (see + :ref:`union-types`), ``Literal``, ``Type``, ``Enum``, ``PathLike``, ``UUID``, + ``Fraction``, ``re.Pattern`` (str only), ``datetime``/``date``/``time`` (ISO + 8601), ``timedelta``, the restricted types of :ref:`restricted-numbers` and :ref:`restricted-strings`, and the path and URL types of :ref:`parsing-paths` and :ref:`parsing-urls`. - ``dict``, ``Mapping``, ``MutableMapping``, ``MappingProxyType``, - ``OrderedDict`` and ``TypedDict`` are supported, but only with ``str`` or - ``int`` keys, see :ref:`dict-items`. + ``OrderedDict``, ``Counter``, ``ChainMap`` and ``TypedDict`` are supported, + but only with ``str`` or ``int`` keys, see :ref:`dict-items`. - ``TypedDict`` accepts ``Required`` and ``NotRequired`` to mark single keys as required or optional, ``ReadOnly`` (PEP `705 @@ -537,7 +539,8 @@ Types can be nested with any complexity. Notes about the support: and pydantic's ``BaseModel`` are supported, even when nested. By default they don't accept subclasses, see :ref:`subclasses-disabled` and :ref:`enable-disable-subclasses`. A dataclass that also inherits from a normal - class does accept subclasses by default. A pydantic model configured with + class does accept subclasses by default. Fields typed ``Final[X]`` or + ``InitVar[X]`` are parsed as ``X``. A pydantic model configured with ``extra`` as ``"allow"`` or ``"ignore"`` accepts keys not in its signature, which are forwarded to the model on instantiation. @@ -689,7 +692,8 @@ part of it can't be validated when: - It failed to resolve, e.g. a missing import or a typo in a postponed annotation. - It is not a type that jsonargparse supports, e.g. a ``TypeVar`` that stands - for nothing, see :ref:`generic-types`. + for nothing (see :ref:`generic-types`) or a variadic tuple like + ``tuple[int, *Ts]``. The debug log gives the reason for each part, see :ref:`logging`. A parameter without a type annotation is shown as ``Untyped`` and behaves the same, see diff --git a/jsonargparse/_completions_jsonschema.py b/jsonargparse/_completions_jsonschema.py index c178a092..1bcd1c57 100644 --- a/jsonargparse/_completions_jsonschema.py +++ b/jsonargparse/_completions_jsonschema.py @@ -5,6 +5,7 @@ import operator import re import uuid +from collections import Counter, abc from enum import Enum from types import ModuleType from typing import Any, Callable, Optional, Tuple, Union @@ -81,7 +82,7 @@ "present in the config, or can be given as a command line argument." ) -basic_type_schemas = { +basic_type_schemas: dict[Any, dict] = { bool: {"type": "boolean"}, int: {"type": "integer"}, float: {"type": "number"}, @@ -90,6 +91,12 @@ dict: {"type": "object"}, list: {"type": "array"}, ModuleType: {"type": "string"}, + # in json, objects are not hashable (arrays become tuples), and numbers, booleans and null are not sized + abc.Hashable: { + "type": ["null", "boolean", "integer", "number", "string", "array"], + "items": {"not": {"type": "object"}}, + }, + abc.Sized: {"type": ["string", "array", "object"]}, } uuid_schema = { @@ -517,6 +524,8 @@ def typehint_schema(self, typehint, action, key: str) -> dict: return self.items_schema(typehint, action, key, {"type": "array"}) if root in mapping_origin_types: args: tuple = getattr(typehint, "__args__", ()) + if root is Counter and args: + args = (args[0], int) # a Counter is subscripted only with the type of its keys if len(args) == 2: values_schema = self.typehint_schema(args[1], action, key) if values_schema: diff --git a/jsonargparse/_namespace.py b/jsonargparse/_namespace.py index 5890d5fe..deffc9f0 100644 --- a/jsonargparse/_namespace.py +++ b/jsonargparse/_namespace.py @@ -85,7 +85,7 @@ def _recreate_branches(data, skip_keys): new_data = data if isinstance(data, (Namespace, dict)) and not isinstance(data, OrderedDict): new_data = type(data)() - for key, val in getattr(data, "__dict__", data).items(): + for key, val in (data.__dict__ if isinstance(data, Namespace) else data).items(): if skip_keys is None or key not in skip_keys: new_data[key] = _recreate_branches(val, skip_keys) provenance = getattr(data, provenance_key, None) diff --git a/jsonargparse/_parameter_resolvers.py b/jsonargparse/_parameter_resolvers.py index 46a54151..fafd0bb7 100644 --- a/jsonargparse/_parameter_resolvers.py +++ b/jsonargparse/_parameter_resolvers.py @@ -12,7 +12,7 @@ from functools import partial, partialmethod from importlib import import_module from types import MethodType -from typing import Any, Literal, Union +from typing import Any, Final, Literal, Union from ._common import ( LoggerProperty, @@ -322,10 +322,20 @@ def get_signature_parameters_and_indexes(component, parent, logger): ) evaluate_postponed_annotations(params, signature_source, parent, logger) stubs = get_stub_types(params, signature_source, parent, logger) + strip_field_qualifiers(params) replace_generic_type_vars(params, parent) return params, args_idx, kwargs_idx, doc_params, stubs +def strip_field_qualifiers(params: ParamList) -> None: + """Removes a top level Final or InitVar, which qualify a field of a dataclass-like type but not its value.""" + for param in params: + if get_typehint_origin(param.annotation) is Final: + param.annotation = param.annotation.__args__[0] + elif isinstance(param.annotation, dataclasses.InitVar): + param.annotation = param.annotation.type + + def replace_generic_type_vars(params: ParamList, parent) -> None: if is_generic_class(parent) and parent.__args__ and getattr(parent.__origin__, "__parameters__", None): from ._typehints import rebuild_typehint_args @@ -1256,6 +1266,7 @@ def get_parameters_from_pydantic_or_attrs( field_names.append(get_field_names(field, name, function_or_class)) # after the attribute names have been used to resolve the annotations evaluate_postponed_annotations(params, function_or_class, None, logger) + strip_field_qualifiers(params) set_param_names_and_aliases(params, field_names) return params diff --git a/jsonargparse/_postponed_annotations.py b/jsonargparse/_postponed_annotations.py index 94e4e299..fd2aee15 100644 --- a/jsonargparse/_postponed_annotations.py +++ b/jsonargparse/_postponed_annotations.py @@ -9,6 +9,7 @@ from types import UnionType from typing import Any, ForwardRef, TypeAlias, TypeVar, Union, get_type_hints +from ._common import get_generic_origin from ._typehints import literal_types, mapping_origin_types, sequence_origin_types, tuple_set_origin_types from ._util import get_typehint_origin @@ -403,8 +404,9 @@ def evaluate_postponed_annotations(params, component, parent, logger): if not (params and any(type_requires_eval(p.annotation) for p in params)): return try: - if is_generated_dataclass_init(component, parent): - types = get_types(parent, logger) + dataclass = get_generic_origin(parent) # a subscripted generic dataclass is resolved from its origin + if is_generated_dataclass_init(component, dataclass): + types = get_types(dataclass, logger) else: types = get_types(component, logger, parent) except Exception as ex: diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py index b9d55e06..cb337870 100644 --- a/jsonargparse/_typehints.py +++ b/jsonargparse/_typehints.py @@ -8,7 +8,7 @@ import sys import typing from argparse import ArgumentError -from collections import OrderedDict, abc, defaultdict, deque +from collections import ChainMap, Counter, OrderedDict, abc, defaultdict, deque from contextlib import contextmanager, suppress from contextvars import ContextVar from copy import deepcopy @@ -29,6 +29,7 @@ ForwardRef, FrozenSet, Iterable, + Iterator, List, Literal, Mapping, @@ -68,6 +69,7 @@ is_instance, is_subclass, is_subclasses_disabled, + is_unpack_typehint, lenient_check, nested_links, parent_parser, @@ -165,12 +167,14 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: Collection, Container, Iterable, + Iterator, Reversible, Sequence, MutableSequence, abc.Collection, abc.Container, abc.Iterable, + abc.Iterator, abc.Reversible, abc.Sequence, abc.MutableSequence, @@ -191,6 +195,10 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: abc.Mapping, abc.MutableMapping, OrderedDict, + Counter, + ChainMap, + abc.Hashable, + abc.Sized, Callable, abc.Callable, ModuleType, @@ -199,7 +207,6 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: NotRequired, ReadOnly, Required, - Unpack, } leaf_types = { @@ -221,12 +228,14 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: Collection, Container, Iterable, + Iterator, Reversible, Sequence, MutableSequence, abc.Collection, abc.Container, abc.Iterable, + abc.Iterator, abc.Reversible, abc.Sequence, abc.MutableSequence, @@ -240,9 +249,12 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: abc.Mapping, abc.MutableMapping, OrderedDict, + Counter, + ChainMap, } sequence_or_mapping_origin_types = sequence_origin_types.union(mapping_origin_types) callable_origin_types = {Callable, abc.Callable} +structural_types = {abc.Hashable, abc.Sized} literal_types = {Literal} _capture_typing_extension_shadows("Literal", root_types, literal_types) @@ -267,9 +279,6 @@ def _capture_typing_extension_shadows(name: str, *collections) -> None: no_extra_items_types = {NoExtraItems} _capture_typing_extension_shadows("NoExtraItems", no_extra_items_types) -unpack_types = {Unpack} -_capture_typing_extension_shadows("Unpack", unpack_types) - subclass_arg_parser: ContextVar = ContextVar("subclass_arg_parser") allow_default_instance: ContextVar = ContextVar("allow_default_instance", default=False) sub_defaults: ContextVar = ContextVar("sub_defaults", default=False) @@ -443,6 +452,10 @@ def is_supported_typehint(typehint, full=False, expanding_aliases=frozenset()): A recursive alias is considered supported where it references itself, since it is supported if the rest is. Aliases are compared without what they are subscripted with, since this can change in each reference. """ + if not isinstance(typehint, abc.Hashable) or is_unpacked(typehint): + # unhashable, e.g. a ParamSpecArgs, can't be looked up in the sets of supported types, and + # unpacked, e.g. *Ts, only makes sense in a variadic tuple, which is not supported either + return False if get_registered_type(typehint) is not None: return True if is_alias_type(typehint): @@ -1062,6 +1075,11 @@ def __repr__(self): Untyped = UntypedType() +def is_unpacked(typehint) -> bool: + """Whether a type hint is unpacked, e.g. ``*Ts``, ``Unpack[Ts]`` or ``*tuple[int, ...]``.""" + return is_unpack_typehint(typehint) or getattr(typehint, "__unpacked__", False) is True + + def accepts_any_value(typehint) -> bool: """Whether a type hint accepts any value, i.e. it does not validate.""" return typehint is object or typehint == Any or isinstance(typehint, UnvalidatedType) @@ -1104,6 +1122,9 @@ def replaced(typehint, reason): # only a tuple, since e.g. types.UnionType and types.GenericAlias have __args__ as a # class level slot descriptor, which is truthy but not the subtypes of an instance if isinstance(args, tuple) and args: + if replace_unsupported and typehint_origin is tuple and any(is_unpacked(a) for a in args): + # a variadic tuple, i.e. its number of elements is not fixed + return replaced(typehint, unsupported_reason) # Subtypes are only validated when the origin is a supported container type. For the # others, e.g. a user defined generic, the subtypes are not used for validation, so # only the unresolved ones are replaced, keeping the type hint as in the source code. @@ -1684,6 +1705,9 @@ def adapt_typehints( val[n] = adapt_typehints(v, subtypehints[0], **adapt_kwargs_n) if typehint_origin is deque: val = list(val) if serialize else deque(val) + elif typehint_origin is abc.Iterator and instantiate_classes: + # only on instantiation, since an iterator would be exhausted by e.g. a dump + val = iter(val) # Dict, Mapping elif typehint_origin in mapping_origin_types: @@ -1698,10 +1722,12 @@ def adapt_typehints( val = {**prev_val, val.key: val.val} else: val = {val.key: val.val} - elif isinstance(val, MappingProxyType): + elif isinstance(val, (MappingProxyType, ChainMap)): val = dict(val) elif not isinstance(val, dict): raise_unexpected_value(f"Expected a {typehint_origin}", val) + if typehint_origin is Counter and subtypehints is not None: + subtypehints = (subtypehints[0], int) # a Counter is subscripted only with the type of its keys if subtypehints is not None: if subtypehints[0] == int: cast = str if serialize else int @@ -1741,8 +1767,8 @@ def adapt_typehints( val[k] = adapt_typehints(v, replace_unvalidatable_typehints(key_type), **adapt_kwargs) if typehint_origin is MappingProxyType and not serialize: val = MappingProxyType(val) - elif typehint_origin is OrderedDict: - val = dict(val) if serialize else OrderedDict(val) + elif typehint_origin in {OrderedDict, Counter, ChainMap}: + val = dict(val) if serialize else typehint_origin(val) # TypedDict Required, NotRequired and ReadOnly elif typehint_origin in typed_dict_key_qualifiers: @@ -1781,6 +1807,15 @@ def adapt_typehints( if not serialize: val = typehint(**val) + # Hashable and Sized, which accept any value that is hashable or that has a length + elif typehint_origin in structural_types: + if typehint_origin is abc.Hashable and not serialize: + val = lists_to_tuples(val) # a list is never hashable, and json and yaml have no tuples + if typehint_origin is abc.Sized and isinstance(val, dict) and "class_path" in val: + raise_unexpected_value("Subclass specs are not supported for Sized", val) + if not is_structural_instance(val, typehint_origin): + raise_unexpected_value(f"Expected a {typehint_origin}", val) + # Callable elif ( typehint_origin in callable_origin_types @@ -1912,7 +1947,7 @@ def adapt_typehints( try: class_path = resolve_class_path_by_name(typehint, val["class_path"]) val_class = import_object(class_path) - if is_instance_or_supports_protocol(val_class, typehint): + if not val.get("init_args") and is_instance_or_supports_protocol(val_class, typehint): return val_class # importable instance if is_protocol(val_class): raise_unexpected_value(f"Expected an instantiatable class, but {val['class_path']} is a protocol") @@ -2237,6 +2272,22 @@ def is_subclass_or_implements_protocol(value, class_type) -> bool: return is_subclass(value, class_type) +def lists_to_tuples(value): + """Returns the value with all its lists, including nested ones, converted to tuples.""" + return tuple(lists_to_tuples(v) for v in value) if isinstance(value, list) else value + + +def is_structural_instance(value, class_type) -> bool: + """Whether a value is an instance of Hashable or Sized, where a tuple is hashable only if all its items are.""" + if class_type is abc.Hashable: + try: + hash(value) + except TypeError: + return False + return True + return isinstance(value, class_type) + + def is_instance_or_supports_protocol(value, class_type): if is_protocol(class_type): # a function implements a callable protocol by itself, any other value by its class @@ -2856,7 +2907,8 @@ def replace_type_vars(typehint): ``type[~T]``. A TypeVar that stands for nothing is left as is, so that it becomes an ``Unvalidated<...>``. """ - if isinstance(typehint, TypeVar): + # hasattr since in python 3.10 a TypeVarTuple and Unpack[Ts] from typing_extensions pretend to be a TypeVar + if isinstance(typehint, TypeVar) and hasattr(typehint, "__constraints__"): return replace_type_var(typehint, in_type_subtype=False) if get_typehint_origin(typehint) in literal_types: return typehint # the args of a Literal are values, not types diff --git a/jsonargparse_tests/test_attrs.py b/jsonargparse_tests/test_attrs.py index 65db05ed..f0846ba8 100644 --- a/jsonargparse_tests/test_attrs.py +++ b/jsonargparse_tests/test_attrs.py @@ -1,11 +1,11 @@ from __future__ import annotations -from typing import List +from typing import Final, List from unittest.mock import patch import pytest -from jsonargparse import Namespace, set_parsing_settings +from jsonargparse import ArgumentError, Namespace, set_parsing_settings from jsonargparse._optionals import attrs_support from jsonargparse_tests.conftest import get_parser_help, skip_if_docstring_parser_unavailable @@ -62,6 +62,10 @@ class AttrsAttrDocsSub(AttrsAttrDocsBase): p2: int = 2 """p2 description""" + @attrs.define + class AttrsFinal: + p1: Final[int] = 1 + @pytest.mark.skipif(not attrs_support, reason="attrs package is required") class TestAttrs: @@ -129,3 +133,10 @@ def test_private_attribute_init_name(self, parser): assert cfg.d == Namespace(why=1, p2="x") init = parser.instantiate(cfg) assert init.d._p2 == "x" + + def test_final_field(self, parser): + parser.add_class_arguments(AttrsFinal, "d") + assert "(type: int, default: 1)" in get_parser_help(parser) + assert parser.parse_args(["--d.p1=2"]).d == Namespace(p1=2) + with pytest.raises(ArgumentError): + parser.parse_args(["--d.p1=x"]) diff --git a/jsonargparse_tests/test_completions_jsonschema.py b/jsonargparse_tests/test_completions_jsonschema.py index c46c1766..8dab9f14 100644 --- a/jsonargparse_tests/test_completions_jsonschema.py +++ b/jsonargparse_tests/test_completions_jsonschema.py @@ -8,17 +8,21 @@ import uuid from abc import ABC, abstractmethod from calendar import Calendar +from collections import Counter from enum import Enum from importlib.util import find_spec from typing import ( Any, Callable, Dict, + Hashable, + Iterator, List, Literal, NamedTuple, Optional, Set, + Sized, Tuple, Type, TypedDict, @@ -353,6 +357,25 @@ def test_container_types(parser): assert properties["set"] == {"type": "array", "uniqueItems": True} +def test_counter_and_iterator_types(parser): + parser.add_argument("--counter", type=Counter[str]) + parser.add_argument("--iterator", type=Iterator[int]) + properties = get_schema(parser)["properties"] + assert properties["counter"] == {"type": "object", "additionalProperties": {"type": "integer"}} + assert properties["iterator"] == {"type": "array", "items": {"type": "integer"}} + + +def test_hashable_and_sized_types(parser): + parser.add_argument("--hashable", type=Hashable) + parser.add_argument("--sized", type=Sized) + properties = get_schema(parser)["properties"] + assert properties["hashable"] == { + "type": ["null", "boolean", "integer", "number", "string", "array"], + "items": {"not": {"type": "object"}}, + } + assert properties["sized"] == {"type": ["string", "array", "object"]} + + GrowingVar = TypeVar("GrowingVar") if type_alias_type: diff --git a/jsonargparse_tests/test_dataclasses.py b/jsonargparse_tests/test_dataclasses.py index 7d985ff3..bb80e665 100644 --- a/jsonargparse_tests/test_dataclasses.py +++ b/jsonargparse_tests/test_dataclasses.py @@ -4,7 +4,7 @@ import dataclasses import json import sys -from typing import Any, Dict, Generic, List, Optional, Tuple, TypeVar, Union +from typing import Any, Dict, Final, Generic, List, Optional, Tuple, TypeVar, Union from unittest.mock import patch import pytest @@ -23,6 +23,7 @@ typing_extensions_import, ) from jsonargparse._signatures import convert_to_dict +from jsonargparse._subcommands import find_action from jsonargparse.typing import PositiveFloat, PositiveInt, restricted_number_type from jsonargparse_tests.conftest import ( get_parse_args_stdout, @@ -650,6 +651,31 @@ def test_generic_dataclass_subclass(parser): assert isinstance(init.x.children[1], GenericChild) +@dataclasses.dataclass +class FinalAndInitVarData(Generic[V]): + init_var: dataclasses.InitVar[V] + fin: Final[int] = 1 + + def __post_init__(self, init_var): + self.from_init_var = init_var + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="python 3.10 fails to resolve a postponed InitVar annotation") +def test_dataclass_final_and_init_var_fields(parser): + parser.add_class_arguments(FinalAndInitVarData[float], "d") + assert find_action(parser, "d.fin")._typehint is int + assert find_action(parser, "d.init_var")._typehint is float + cfg = parser.parse_args(["--d.fin=2", "--d.init_var=3"]) + assert cfg.d == Namespace(init_var=3.0, fin=2) + for args in [["--d.fin=x", "--d.init_var=3"], ["--d.init_var=x"]]: + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(args) + ctx.match("Expected a " if "--d.fin=x" in args else "Expected a ") + init = parser.instantiate(cfg) + assert init.d.fin == 2 + assert init.d.from_init_var == 3.0 + + @dataclasses.dataclass class UnionData: data_a: int = 1 diff --git a/jsonargparse_tests/test_subclasses.py b/jsonargparse_tests/test_subclasses.py index 461a7cfa..6391e219 100644 --- a/jsonargparse_tests/test_subclasses.py +++ b/jsonargparse_tests/test_subclasses.py @@ -641,6 +641,14 @@ def test_importable_instances(parser): assert dump == {"dtype": f"{__name__}.float32"} +def test_importable_instance_with_init_args(parser): + parser.add_argument("--dtype", type=dtype) + value = {"class_path": f"{__name__}.float32", "init_args": {"a": 1}} + with pytest.raises(ArgumentError) as ctx: + parser.parse_args([f"--dtype={json.dumps(value)}"]) + ctx.match("float32 does not correspond to a subclass of dtype") + + calendar_instance = Calendar(firstweekday=3) diff --git a/jsonargparse_tests/test_typehints.py b/jsonargparse_tests/test_typehints.py index 6aab9e3b..46c5dc40 100644 --- a/jsonargparse_tests/test_typehints.py +++ b/jsonargparse_tests/test_typehints.py @@ -8,7 +8,7 @@ import sys import time import uuid -from collections import OrderedDict, abc, deque, namedtuple +from collections import ChainMap, Counter, OrderedDict, abc, deque, namedtuple from contextlib import contextmanager from dataclasses import dataclass, field from datetime import date @@ -21,6 +21,7 @@ AbstractSet, Annotated, Any, + Awaitable, Callable, Collection, Container, @@ -40,6 +41,7 @@ NewType, NoReturn, Optional, + ParamSpec, Protocol, Reversible, Sequence, @@ -63,6 +65,7 @@ typing_extensions_import, typing_extensions_support, ) +from jsonargparse._subcommands import find_action from jsonargparse._typehints import ( ActionTypeHint, NoExtraItems, @@ -812,6 +815,48 @@ def test_tuples_nested_ellipsis(parser): assert (("foo", "bar"), ((1, 2.02), (3, 3.09))) == cfg.tuple +TypeVarTuple = typing_extensions_import("TypeVarTuple") +skip_if_no_variadic_generics = pytest.mark.skipif( + not (Unpack and TypeVarTuple), reason="Unpack and TypeVarTuple introduced in python 3.11 or in typing_extensions" +) + +if Unpack and TypeVarTuple: + Ts = TypeVarTuple("Ts") + + def function_variadic_tuples( + p1: Tuple[int, Unpack[Ts]] = (0, "a"), + p2: Tuple[int, Unpack[Tuple[str, ...]]] = (0, "a"), + *args: Unpack[Ts], + ): + return p1, p2, args # pragma: no cover + + +@skip_if_no_variadic_generics +def test_variadic_tuples_unvalidated(parser): + # the number of elements of a variadic tuple is not fixed, thus it is not validated + parser.add_function_arguments(function_variadic_tuples, "fn") + p1, p2 = Tuple[int, Unpack[Ts]], Tuple[int, Unpack[Tuple[str, ...]]] + assert find_action(parser, "fn.p1")._typehint == UnvalidatedType(p1) + assert find_action(parser, "fn.p2")._typehint == UnvalidatedType(p2) + assert find_action(parser, "fn.args")._typehint == list[UnvalidatedType(Unpack[Ts])] + cfg = parser.parse_args(['--fn.p1=[1, "a", 2]', '--fn.p2=[1, "a", "b"]', '--fn.args=[1, "a"]']) + assert cfg.fn == Namespace(p1=[1, "a", 2], p2=[1, "a", "b"], args=[1, "a"]) + + +@skip_if_no_variadic_generics +def test_variadic_tuple_unsupported_as_argument_type(): + with pytest.raises(ValueError, match="Unsupported type hint"): + ActionTypeHint(typehint=Tuple[int, Unpack[Tuple[str, ...]]]) + + +@pytest.mark.skipif(sys.version_info < (3, 11), reason="star unpacking in subscripts introduced in python 3.11") +def test_star_unpacked_tuple_unvalidated(): + unpacked = next(iter(tuple[str, ...])) # i.e. *tuple[str, ...], which is not valid syntax in python 3.10 + typehint = tuple[int, unpacked] + assert replace_unvalidatable_typehints(typehint) == UnvalidatedType(typehint) + assert replace_unvalidatable_typehints(unpacked) == UnvalidatedType(unpacked) + + def test_tuple_union(parser, tmp_cwd): parser.add_argument("--tuple", type=Tuple[Union[int, EnumABC], Path_fc, NotEmptyStr]) cfg = parser.parse_args(['--tuple=[2, "a", "b"]']) @@ -1493,7 +1538,7 @@ def test_namedtuple_class_field_instantiate(parser): Vector = NewType("Vector", List[float]) NestedUserId = NewType("NestedUserId", UserId) CalendarType = NewType("CalendarType", calendar.Calendar) -UnsupportedNewType = NewType("UnsupportedNewType", Iterator[int]) # type: ignore[misc] +UnsupportedNewType = NewType("UnsupportedNewType", Awaitable[int]) # type: ignore[misc] def test_new_type(parser): @@ -1654,7 +1699,7 @@ def test_unsubscripted_sequence_alias(parser, alias, expected): assert parser.dump(cfg, format="json_compact") == '{"x":[1,2]}' -@pytest.mark.parametrize("alias", [Dict, Mapping, MutableMapping], ids=str) +@pytest.mark.parametrize("alias", [Dict, Mapping, MutableMapping, Counter, ChainMap], ids=str) def test_unsubscripted_mapping_alias(parser, alias): parser.add_argument("--x", type=alias) cfg = parser.parse_args(['--x={"a": 1}']) @@ -1691,12 +1736,51 @@ def test_unsubscripted_alias_signature_parameter(parser): def test_hashable(parser, hashable): parser.add_argument("--x", type=hashable) assert parser.parse_args(["--x=abc"]).x == "abc" + assert "--x.help" not in get_parser_help(parser) # not a subclass type + + +@pytest.mark.parametrize("hashable", [Hashable, abc.Hashable], ids=str) +def test_hashable_list_to_tuple(parser, hashable): + # a list is never hashable, and json and yaml have no tuples + parser.add_argument("--x", type=hashable) + cfg = parser.parse_args(['--x=[1, ["a", "b"]]']) + assert cfg.x == (1, ("a", "b")) + dump = parser.dump(cfg, format="json_compact") + assert dump == '{"x":[1,["a","b"]]}' + assert parser.parse_string(dump).x == (1, ("a", "b")) + + +@pytest.mark.parametrize("value", [{"a": 1}, {"class_path": "calendar.Calendar"}, [1, {"a": 1}]], ids=str) +@pytest.mark.parametrize("hashable", [Hashable, abc.Hashable], ids=str) +def test_hashable_unhashable_value(parser, hashable, value): + parser.add_argument("--x", type=hashable) + with pytest.raises(ArgumentError) as ctx: + parser.parse_object({"x": value}) + ctx.match("Expected a ") @pytest.mark.parametrize("sized", [Sized, abc.Sized], ids=str) def test_sized(parser, sized): parser.add_argument("--x", type=sized) assert parser.parse_args(["--x=[1, 2]"]).x == [1, 2] + assert "--x.help" not in get_parser_help(parser) # not a subclass type + + +@pytest.mark.parametrize("sized", [Sized, abc.Sized], ids=str) +def test_sized_subclass_spec_rejected(parser, sized): + # reserved, so that supporting subclass specs in the future is not a breaking change + parser.add_argument("--x", type=sized) + with pytest.raises(ArgumentError) as ctx: + parser.parse_object({"x": {"class_path": "collections.UserList", "init_args": {"initlist": [1]}}}) + ctx.match("Subclass specs are not supported for Sized") + + +@pytest.mark.parametrize("sized", [Sized, abc.Sized], ids=str) +def test_sized_unsized_value(parser, sized): + parser.add_argument("--x", type=sized) + with pytest.raises(ArgumentError) as ctx: + parser.parse_object({"x": 1}) + ctx.match("Expected a ") @pytest.mark.parametrize("hashable", [Hashable, abc.Hashable], ids=str) @@ -1832,7 +1916,7 @@ def test_typeddict_key_bound_typevar(parser): class UnsupportedKeyOptions(TypedDict, total=False): supported: int - unsupported: Iterator[int] + unsupported: Awaitable[int] def test_typeddict_key_unsupported_type(parser): @@ -1842,7 +1926,7 @@ def test_typeddict_key_unsupported_type(parser): assert parser.parse_args(['--options={"unsupported": [1]}']).options == {"unsupported": [1]} help_str = get_parse_args_stdout(parser, ["--options.help"]) assert "--options.unsupported UNSUPPORTED" in help_str - assert "(type: Unvalidated)" in help_str + assert "(type: Unvalidated)" in help_str def test_subscripted_non_generic_typeddict(parser): @@ -2519,8 +2603,9 @@ def test_signature_required_param_with_default(parser, wrappers_module): @pytest.mark.skipif(not Unpack, reason="Unpack introduced in python 3.11 or backported in typing_extensions") -def test_unpack_support(parser): - assert ActionTypeHint.is_supported_typehint(Unpack[Any]) +def test_unpack_not_supported_by_itself(parser): + # Unpack is only supported for **kwargs, whose TypedDict is expanded into parameters + assert not ActionTypeHint.is_supported_typehint(Unpack[Any]) if Unpack: # and Required and NotRequired @@ -2791,6 +2876,56 @@ def test_ordered_dict(parser): assert parser.dump(cfg, format="yaml") == "odict:\n a: 1\n b: 2\n" +def test_counter(parser): + parser.add_argument("--counter", type=Counter[str]) + assert "(type: Counter[str], default: null)" in get_parser_help(parser) + cfg = parser.parse_args(['--counter={"a":2, "b":1}']) + assert isinstance(cfg.counter, Counter) + assert cfg.counter == Counter({"a": 2, "b": 1}) + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--counter={"a":"x"}']) + ctx.match("Expected a ") + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(["--counter=[a, a]"]) + ctx.match("Expected a ") + assert parser.dump(cfg, format="json_compact") == '{"counter":{"a":2,"b":1}}' + + +def test_chain_map(parser): + default = ChainMap({"a": 1}, {"b": 2}) + parser.add_argument("--chain", type=ChainMap[str, int], default=default) + assert parser.get_defaults().chain == {"a": 1, "b": 2} + cfg = parser.parse_args(['--chain={"c":3}']) + assert isinstance(cfg.chain, ChainMap) + assert cfg.chain == ChainMap({"c": 3}) + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--chain={"c":"x"}']) + ctx.match("Expected a ") + assert parser.dump(cfg, format="json_compact") == '{"chain":{"c":3}}' + + +@pytest.mark.parametrize("iterator", [Iterator, abc.Iterator], ids=str) +def test_iterator(parser, iterator): + parser.add_argument("--iter", type=iterator[int]) + cfg = parser.parse_args(["--iter=[1, 2]"]) + # kept as a list, since an iterator would be exhausted by e.g. a dump + assert cfg.iter == [1, 2] + assert parser.dump(cfg, format="json_compact") == '{"iter":[1,2]}' + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--iter=["x"]']) + ctx.match("Expected a ") + init = parser.instantiate(cfg) + assert isinstance(init.iter, abc.Iterator) + assert list(init.iter) == [1, 2] + + +def test_dict_default_counter(parser): + parser.add_argument("--dict", type=dict, default=Counter({"a": 2})) + cfg = parser.parse_args([]) + assert isinstance(cfg.dict, Counter) + assert cfg.dict == {"a": 2} + + def test_dict_default_ordered_dict(parser): parser.add_argument("--dict", type=dict, default=OrderedDict({"a": 1})) defaults = parser.get_defaults() @@ -3845,6 +3980,19 @@ def __init__(self, p1: int = 1): self.p1 = p1 # pragma: no cover +P = ParamSpec("P") + + +def function_param_spec_args(p1: P.args): # type: ignore[valid-type] + return p1 # pragma: no cover + + +def test_unhashable_type_unvalidated(parser): + parser.add_function_arguments(function_param_spec_args, "fn") + assert find_action(parser, "fn.p1")._typehint == UnvalidatedType(P.args) + assert parser.parse_args(["--fn.p1=x"]).fn.p1 == "x" + + def test_unvalidated_type_repr(): assert repr(unvalidated_var) == "Unvalidated" assert repr(UnvalidatedType(NoReturn)) == "Unvalidated" From 9b3eec5d4b04f72d9b2938f06cac6f7d6cfa0f46 Mon Sep 17 00:00:00 2001 From: Mauricio Villegas <5780272+mauvilsa@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:37:53 +0200 Subject: [PATCH 2/2] Address comments --- jsonargparse/_completions_jsonschema.py | 23 +++++++++++------- jsonargparse/_typehints.py | 10 ++++---- .../test_completions_jsonschema.py | 24 +++++++++++++++---- jsonargparse_tests/test_typehints.py | 16 +++++++++++++ 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/jsonargparse/_completions_jsonschema.py b/jsonargparse/_completions_jsonschema.py index 1bcd1c57..2c4f1463 100644 --- a/jsonargparse/_completions_jsonschema.py +++ b/jsonargparse/_completions_jsonschema.py @@ -91,12 +91,8 @@ dict: {"type": "object"}, list: {"type": "array"}, ModuleType: {"type": "string"}, - # in json, objects are not hashable (arrays become tuples), and numbers, booleans and null are not sized - abc.Hashable: { - "type": ["null", "boolean", "integer", "number", "string", "array"], - "items": {"not": {"type": "object"}}, - }, - abc.Sized: {"type": ["string", "array", "object"]}, + # in json, numbers, booleans and null are not sized, and subclass specs are rejected + abc.Sized: {"type": ["string", "array", "object"], "not": {"type": "object", "required": ["class_path"]}}, } uuid_schema = { @@ -497,6 +493,8 @@ def typehint_schema(self, typehint, action, key: str) -> dict: return dict(uuid_schema) if is_sentinel(typehint): return {"const": get_import_path(typehint)} + if root is abc.Hashable: + return self.hashable_ref(key) if typehint in basic_type_schemas: return dict(basic_type_schemas[typehint]) registered = get_registered_type(typehint) @@ -524,8 +522,8 @@ def typehint_schema(self, typehint, action, key: str) -> dict: return self.items_schema(typehint, action, key, {"type": "array"}) if root in mapping_origin_types: args: tuple = getattr(typehint, "__args__", ()) - if root is Counter and args: - args = (args[0], int) # a Counter is subscripted only with the type of its keys + if root is Counter: + args = (args[0] if args else Any, int) # a Counter is subscripted only with the type of its keys if len(args) == 2: values_schema = self.typehint_schema(args[1], action, key) if values_schema: @@ -537,6 +535,15 @@ def typehint_schema(self, typehint, action, key: str) -> dict: return self.class_ref(typehint, action, key, subclass=False) return {} + def hashable_ref(self, key: str) -> dict: + """References the definition of Hashable, which in json is anything but objects, including nested in arrays.""" + + def build_def(kwargs: dict) -> dict: + json_types = ["null", "boolean", "integer", "number", "string", "array"] + return {"type": json_types, "items": self.hashable_ref(key)} + + return self.def_ref(("hashable",), self.def_name(abc.Hashable), key, {}, build_def) + def alias_schema(self, alias, action, key: str) -> dict: """Describes a type alias, which when recursive is a definition that has variants like the ones of classes. diff --git a/jsonargparse/_typehints.py b/jsonargparse/_typehints.py index cb337870..178a9c44 100644 --- a/jsonargparse/_typehints.py +++ b/jsonargparse/_typehints.py @@ -1714,7 +1714,8 @@ def adapt_typehints( if not serialize and not instantiate_classes: validate_subclass_spec_in_mapping(val, typehint, subtypehints, sub_add_kwargs, logger) if isinstance(val, NestedArg): - if isinstance(prev_val, dict): + if isinstance(prev_val, abc.Mapping): + prev_val = dict(prev_val) if isinstance(val.key, str) and "." in val.key: key_prefix, key_suffix = val.key.split(".", 1) val = {**prev_val, key_prefix: {key_suffix: val.val}} @@ -1726,8 +1727,9 @@ def adapt_typehints( val = dict(val) elif not isinstance(val, dict): raise_unexpected_value(f"Expected a {typehint_origin}", val) - if typehint_origin is Counter and subtypehints is not None: - subtypehints = (subtypehints[0], int) # a Counter is subscripted only with the type of its keys + if typehint_origin is Counter: + # a Counter is subscripted only with the type of its keys, and its values are always int + subtypehints = (subtypehints[0] if subtypehints else Any, int) if subtypehints is not None: if subtypehints[0] == int: cast = str if serialize else int @@ -1746,7 +1748,7 @@ def adapt_typehints( else: kwargs = adapt_kwargs.copy() if kwargs.get("prev_val"): - if isinstance(kwargs["prev_val"], dict): + if isinstance(kwargs["prev_val"], abc.Mapping): kwargs["prev_val"] = kwargs["prev_val"].get(k) else: kwargs["prev_val"] = None diff --git a/jsonargparse_tests/test_completions_jsonschema.py b/jsonargparse_tests/test_completions_jsonschema.py index 8dab9f14..94ccb499 100644 --- a/jsonargparse_tests/test_completions_jsonschema.py +++ b/jsonargparse_tests/test_completions_jsonschema.py @@ -359,21 +359,37 @@ def test_container_types(parser): def test_counter_and_iterator_types(parser): parser.add_argument("--counter", type=Counter[str]) + parser.add_argument("--counter_any", type=Counter) parser.add_argument("--iterator", type=Iterator[int]) properties = get_schema(parser)["properties"] assert properties["counter"] == {"type": "object", "additionalProperties": {"type": "integer"}} + assert properties["counter_any"] == {"type": "object", "additionalProperties": {"type": "integer"}} assert properties["iterator"] == {"type": "array", "items": {"type": "integer"}} def test_hashable_and_sized_types(parser): parser.add_argument("--hashable", type=Hashable) parser.add_argument("--sized", type=Sized) - properties = get_schema(parser)["properties"] - assert properties["hashable"] == { + schema = get_schema(parser) + assert schema["properties"]["hashable"] == {"$ref": "#/$defs/Hashable"} + assert schema["$defs"]["Hashable"] == { "type": ["null", "boolean", "integer", "number", "string", "array"], - "items": {"not": {"type": "object"}}, + "items": {"$ref": "#/$defs/Hashable"}, + } + assert schema["properties"]["sized"] == { + "type": ["string", "array", "object"], + "not": {"type": "object", "required": ["class_path"]}, } - assert properties["sized"] == {"type": ["string", "array", "object"]} + + +@skip_if_jsonschema_unavailable +def test_hashable_and_sized_schemas_reject_unparsable(parser): + parser.add_argument("--hashable", type=Hashable) + parser.add_argument("--sized", type=Sized) + schema = get_schema(parser) + validate(schema, {"hashable": [[1, "a"]], "sized": {"a": 1}}) + assert iter_errors(schema, {"hashable": [[{"a": 1}]]}) + assert iter_errors(schema, {"sized": {"class_path": "calendar.Calendar"}}) GrowingVar = TypeVar("GrowingVar") diff --git a/jsonargparse_tests/test_typehints.py b/jsonargparse_tests/test_typehints.py index 46c5dc40..fc303f23 100644 --- a/jsonargparse_tests/test_typehints.py +++ b/jsonargparse_tests/test_typehints.py @@ -2891,6 +2891,15 @@ def test_counter(parser): assert parser.dump(cfg, format="json_compact") == '{"counter":{"a":2,"b":1}}' +def test_counter_unsubscripted(parser): + parser.add_argument("--counter", type=Counter) + cfg = parser.parse_args(['--counter={"a":2}']) + assert cfg.counter == Counter({"a": 2}) + with pytest.raises(ArgumentError) as ctx: + parser.parse_args(['--counter={"a":"x"}']) + ctx.match("Expected a ") + + def test_chain_map(parser): default = ChainMap({"a": 1}, {"b": 2}) parser.add_argument("--chain", type=ChainMap[str, int], default=default) @@ -2904,6 +2913,13 @@ def test_chain_map(parser): assert parser.dump(cfg, format="json_compact") == '{"chain":{"c":3}}' +def test_chain_map_nested_arg_keeps_other_keys(parser): + parser.add_argument("--chain", type=ChainMap[str, int]) + cfg = parser.parse_args(['--chain={"a":1, "b":2}', "--chain.a=3"]) + assert isinstance(cfg.chain, ChainMap) + assert cfg.chain == {"a": 3, "b": 2} + + @pytest.mark.parametrize("iterator", [Iterator, abc.Iterator], ids=str) def test_iterator(parser, iterator): parser.add_argument("--iter", type=iterator[int])