diff --git a/mypy/typeshed/stubs/librt/librt/vecs.pyi b/mypy/typeshed/stubs/librt/librt/vecs.pyi new file mode 100644 index 0000000000000..a4e8c88e0e596 --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/vecs.pyi @@ -0,0 +1,21 @@ +from typing import TypeVar, Generic, Iterable, Iterator, overload +from mypy_extensions import i64 + +T = TypeVar("T") + +class vec(Generic[T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, items: Iterable[T], /) -> None: ... + def __len__(self) -> i64: ... + @overload + def __getitem__(self, i: i64, /) -> T: ... + @overload + def __getitem__(self, i: slice, /) -> vec[T]: ... + def __setitem__(self, i: i64, o: T, /) -> None: ... + def __iter__(self) -> Iterator[T]: ... + +def append(v: vec[T], o: T, /) -> vec[T]: ... +def remove(v: vec[T], o: T, /) -> vec[T]: ... +def pop(v: vec[T], i: i64 = -1, /) -> tuple[vec[T], T]: ... diff --git a/mypyc/build.py b/mypyc/build.py index ef5d00fca1457..c2c38eeaa269e 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -104,6 +104,20 @@ class ModDesc(NamedTuple): ], ["base64"], ), + ModDesc( + "librt.vecs", + [ + "vecs/librt_vecs.c", + "vecs/vec_i64.c", + "vecs/vec_i32.c", + "vecs/vec_i16.c", + "vecs/vec_u8.c", + "vecs/vec_float.c", + "vecs/vec_bool.c", + ], + ["vecs/librt_vecs.h", "vecs/vec_template.c"], + ["vecs"], + ), ] try: diff --git a/mypyc/ir/deps.py b/mypyc/ir/deps.py index 1ca709748eeff..7c52520e274b9 100644 --- a/mypyc/ir/deps.py +++ b/mypyc/ir/deps.py @@ -48,6 +48,7 @@ def get_header(self) -> str: LIBRT_STRINGS: Final = Capsule("librt.strings") LIBRT_BASE64: Final = Capsule("librt.base64") +LIBRT_VECS: Final = Capsule("librt.vecs") BYTES_EXTRA_OPS: Final = SourceDep("bytes_extra_ops.c") BYTES_WRITER_EXTRA_OPS: Final = SourceDep("byteswriter_extra_ops.c") diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 964b56e4b5623..9b477e457d758 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -132,5 +132,19 @@ def run(self) -> None: include_dirs=[".", "base64"], extra_compile_args=cflags, ), + Extension( + "librt.vecs", + [ + "vecs/librt_vecs.c", + "vecs/vec_i64.c", + "vecs/vec_i32.c", + "vecs/vec_i16.c", + "vecs/vec_u8.c", + "vecs/vec_float.c", + "vecs/vec_bool.c", + ], + include_dirs=[".", "vecs"], + extra_compile_args=cflags, + ), ] ) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c new file mode 100644 index 0000000000000..bbbefd90cf92b --- /dev/null +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -0,0 +1,916 @@ +// Definition of the mypyc.vecs module, which implements 'vec' and related functionality. +// +// NOTE: This is still experimental and work in progress. +// +// vec[t] is a sequence type that is optimized for use in mypyc-compiled code. +// Performance in interpreted code is a secondary concern. It's primarily optimized +// to provide fast item get/set operations and fast len(), especially for primitive +// value item types, such as 'i32', 'float' or 'bool'. +// +// Example: +// +// from mypy_extensions import i64 +// +// v = vec[i64]() +// v = append(v, 123) +// print(v[0]) +// v[-1] = 234 +// +// Key properties: +// +// * In native code, a vec instance is an immutable value type with two fields +// (length, item buffer). +// * The item buffer can only store types of a specific type (enforced at runtime). +// * There are specialized item buffer types for certain primitive value item types, +// such as 'i32', 'float' and 'bool'. These use a packed value encoding. +// * Since vec values are immutable (only the buffer is mutable), any operation +// that changes the length (such as append) must return the updated vec value. +// * The buffer often a has higher capacity than the length of a vec. Append +// operations only allocate a new buffer if the capacity is exhausted, for O(1) +// amortized time complexity for appends. +// * Vec values can be boxed to PyObject *; this happens transparently and uses +// a wrapper object. +// * The item type can be a class type, or 't | None' if t is a class type. However, +// optional value item types such as 'float | None' are not supported, and 'int' +// is not valid as item type -- a native integer type such as `i64` or `u8` must +// be used instead. +// * Nested vecs are also supported (e.g. vec[vec[i32]]). +// * All features are also available in interpreted code, but often at some +// performance cost compared to using plain lists. +// +// Implementation summary: +// +// * In a non-native context, instances are constructed via a temporary VecGenericAlias +// object, which is the result of 'vec[]' (indexing the 'vec' type). +// * The module exports a C API via a capsule that mypyc compiled code uses for most +// operations. +// * There are multiple C structs that represent vec value objects with different item +// types, such as VecI64 and VecT. The latter is for arbitrary reference objects +// ('PyObject *' items internally). +// * There are also multiple buffer types, such as VecI64Buf and VecTBuf. +// * Similarly, there are multiple Python wrapper object types for boxed vecs, such as +// VecI64Type and VecTType. +// * An empty vec in compiled code can use a NULL pointer to represent the buffer, +// saving an object allocation in a common use case. +// +// Motivation: +// * A vec with value type items, such as vec[i64] or vec[vec[i32]], is very efficient +// compared to a list object, since values are not boxed, there is no need for runtime +// type checks on read, and index overflow checks are very fast due to the immutable +// value encoding. +// * A vec with a reference item type, such as vec[str], is often faster than a list, +// since there is no need for a cast on item read and the index checks are faster +// (among other things), but here the benefit is much smaller compared to using a +// value item type. +// +// Summary of files: +// * vec_t.c implements vec[t] and vec[t | None] for reference types +// * vec_nested.c implements nested vecs, such as vec[vec[str]] +// * vec_template.c is #included in multiples files (e.g. vec_i64.c) to produce +// specialized code for vecs with value item types (e.g. vec[i64]) +// * This file defines the C extension and has some shared code + +#define PY_SSIZE_T_CLEAN +#include +#include "librt_vecs.h" + +#ifdef MYPYC_EXPERIMENTAL + +PyTypeObject *LibRTVecs_I64TypeObj; +PyTypeObject *LibRTVecs_I32TypeObj; +PyTypeObject *LibRTVecs_I16TypeObj; +PyTypeObject *LibRTVecs_U8TypeObj; + + +// vec generic alias +// +// Used for the result of vec[t] (indexing) in interpreted context that must preserve +// knowledge of 't'. These aren't real types. This only supports constructing instances. +typedef struct { + PyObject_HEAD + // Tagged pointer to PyTypeObject *, lowest bit set for optional item type + // Can also be one of magic VEC_ITEM_TYPE_* constants + size_t item_type; + size_t depth; // Number of nested VecNested or VecT types +} VecGenericAlias; + +static PyObject *vec_generic_alias_call(PyObject *self, PyObject *args, PyObject *kw) +{ + PyErr_SetString(PyExc_NotImplementedError, + "non-specialized vec construction not yet supported"); + return NULL; +} + +static int +VecGenericAlias_traverse(VecGenericAlias *self, visitproc visit, void *arg) +{ + if (!Vec_IsMagicItemType(self->item_type)) + Py_VISIT((PyObject *)(self->item_type & ~1)); + return 0; +} + +static void +VecGenericAlias_dealloc(VecGenericAlias *self) +{ + if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { + Py_DECREF((PyObject *)(self->item_type & ~1)); + self->item_type = 0; + } + PyObject_GC_Del(self); +} + +PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { + PyObject *item = NULL; + PyObject *result = NULL; + + if (depth == 0) { + if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { + item = PyUnicode_FromFormat("i64"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_U8) { + item = PyUnicode_FromFormat("u8"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_FLOAT) { + item = PyUnicode_FromFormat("float"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_I32) { + item = PyUnicode_FromFormat("i32"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_I16) { + item = PyUnicode_FromFormat("i16"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_BOOL) { + item = PyUnicode_FromFormat("bool"); + } else { + item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); + if (item == NULL) { + return NULL; + } + if (item_type & 1) { + PyObject *optional_item = PyUnicode_FromFormat("%U | None", item); + Py_DECREF(item); + if (optional_item == NULL) { + return NULL; + } + item = optional_item; + } + } + } else { + item = Vec_TypeToStr(item_type, depth - 1); + } + + if (item == NULL) { + return NULL; + } + + result = PyUnicode_FromFormat("vec[%U]", item); + Py_DECREF(item); + return result; +} + +PyObject *VecGenericAlias_repr(PyObject *self) { + PyObject *l = NULL; + PyObject *prefix = NULL; + PyObject *suffix = NULL; + PyObject *sep = NULL; + PyObject *type_str = NULL; + PyObject *result = NULL; + + l = PyList_New(0); + if (l == NULL) goto error; + + prefix = PyUnicode_FromString(""); + if (suffix == NULL) goto error; + + sep = PyUnicode_FromString(""); + if (sep == NULL) goto error; + + if (PyList_Append(l, prefix) < 0) goto error; + + VecGenericAlias *v = (VecGenericAlias *)self; + type_str = Vec_TypeToStr(v->item_type, v->depth); + if (type_str == NULL) goto error; + + if (PyList_Append(l, type_str) < 0) goto error; + if (PyList_Append(l, suffix) < 0) goto error; + + result = PyUnicode_Join(sep, l); + +error: + Py_XDECREF(l); + Py_XDECREF(prefix); + Py_XDECREF(suffix); + Py_XDECREF(sep); + Py_XDECREF(type_str); + return result; +} + +PyTypeObject VecGenericAliasType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec_generic_alias", + .tp_basicsize = sizeof(VecGenericAlias), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_call = vec_generic_alias_call, + .tp_traverse = (traverseproc)VecGenericAlias_traverse, + .tp_dealloc = (destructor)VecGenericAlias_dealloc, + .tp_repr = (reprfunc)VecGenericAlias_repr, +}; + + +// The 'vec' type +// +// This cannot be instantiated, and it's only used for isinstance and indexing: vec[T]. + +typedef struct { + PyObject_HEAD +} Vec; + +static PyObject *extract_optional_item(PyObject *item) { + PyObject *args = PyObject_GetAttrString(item, "__args__"); + if (args == NULL) { + PyErr_Clear(); + return NULL; + } + if (!PyTuple_CheckExact(args)) + goto error; + if (PyTuple_GET_SIZE(args) != 2) + goto error; + PyObject *item0 = PyTuple_GET_ITEM(args, 0); + PyObject *item1 = PyTuple_GET_ITEM(args, 1); + if (item0 == (PyObject *)Py_None->ob_type) { + Py_DECREF(args); + return item1; + } else if (item1 == (PyObject *)Py_None->ob_type) { + Py_DECREF(args); + return item0; + } + error: + Py_DECREF(args); + return NULL; +} + +// Evaluation vec[] in interpreted code and produce a generic alias object. +static PyObject *vec_class_getitem(PyObject *type, PyObject *item) +{ + if (item == (PyObject *)LibRTVecs_I64TypeObj) { + Py_INCREF(&VecI64Type); + return (PyObject *)&VecI64Type; + } else if (item == (PyObject *)LibRTVecs_U8TypeObj) { + Py_INCREF(&VecU8Type); + return (PyObject *)&VecU8Type; + } else if (item == (PyObject *)&PyFloat_Type) { + Py_INCREF(&VecFloatType); + return (PyObject *)&VecFloatType; + } else if (item == (PyObject *)LibRTVecs_I32TypeObj) { + Py_INCREF(&VecI32Type); + return (PyObject *)&VecI32Type; + } else if (item == (PyObject *)LibRTVecs_I16TypeObj) { + Py_INCREF(&VecI16Type); + return (PyObject *)&VecI16Type; + } else if (item == (PyObject *)&PyBool_Type) { + Py_INCREF(&VecBoolType); + return (PyObject *)&VecBoolType; + } else { + size_t depth = 0; + size_t item_type; + int optional = 0; + if (!PyObject_TypeCheck(item, &PyType_Type)) { + PyObject *it = extract_optional_item(item); + if (it != NULL) { + optional = 1; + item = it; + // Check if this is a specialized value type that doesn't support None + if (item == (PyObject *)LibRTVecs_I64TypeObj || + item == (PyObject *)LibRTVecs_U8TypeObj || + item == (PyObject *)&PyFloat_Type || + item == (PyObject *)LibRTVecs_I32TypeObj || + item == (PyObject *)LibRTVecs_I16TypeObj || + item == (PyObject *)&PyBool_Type) { + PyErr_SetString(PyExc_TypeError, + "vec does not support optional value types (use vec[object] instead)"); + return NULL; + } + } + if (item->ob_type == &VecGenericAliasType) { + if (optional) { + PyErr_SetString(PyExc_TypeError, "optional type not expected in vec[...]"); + return NULL; + } + VecGenericAlias *p = (VecGenericAlias *)item; + item_type = p->item_type; + depth = p->depth + 1; + } else if (!PyObject_TypeCheck(item, &PyType_Type)) { + PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); + return NULL; + } else { + item_type = (size_t)item | optional; + } + } else { + if (item == (PyObject *)&VecI64Type) { + item_type = VEC_ITEM_TYPE_I64; + depth = 1; + } else if (item == (PyObject *)&VecU8Type) { + item_type = VEC_ITEM_TYPE_U8; + depth = 1; + } else if (item == (PyObject *)&VecFloatType) { + item_type = VEC_ITEM_TYPE_FLOAT; + depth = 1; + } else if (item == (PyObject *)&VecI32Type) { + item_type = VEC_ITEM_TYPE_I32; + depth = 1; + } else if (item == (PyObject *)&VecI16Type) { + item_type = VEC_ITEM_TYPE_I16; + depth = 1; + } else if (item == (PyObject *)&VecBoolType) { + item_type = VEC_ITEM_TYPE_BOOL; + depth = 1; + } else { + item_type = (size_t)item; + } + } + if (item == (PyObject *)&PyLong_Type) { + PyErr_Format(PyExc_ValueError, "unsupported type in vec[%s] (use i64, i32, i16, or u8)", + ((PyTypeObject *)item)->tp_name); + return NULL; + } + VecGenericAlias *p; + p = PyObject_GC_New(VecGenericAlias, &VecGenericAliasType); + if (p == NULL) + return NULL; + Py_INCREF(item); + p->item_type = item_type; + p->depth = depth; + PyObject_GC_Track(p); + return (PyObject *)p; + } +} + +static PyMethodDef vec_methods[] = { + {"__class_getitem__", vec_class_getitem, METH_O|METH_CLASS, NULL}, + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + +PyTypeObject VecType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec", + .tp_basicsize = sizeof(Vec), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_methods = vec_methods, +}; + +// Common helpers + +PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose) { + PyObject *l = NULL; + PyObject *prefix = NULL; + PyObject *mid = NULL; + PyObject *suffix = NULL; + PyObject *sep = NULL; + PyObject *comma = NULL; + PyObject *result = NULL; + + l = PyList_New(0); + if (l == NULL) goto error; + + sep = PyUnicode_FromString(""); + if (sep == NULL) goto error; + + comma = PyUnicode_FromString(", "); + if (comma == NULL) goto error; + + if (verbose) { + prefix = Vec_TypeToStr(item_type, depth); + if (prefix == NULL) goto error; + + mid = PyUnicode_FromString("(["); + if (mid == NULL) goto error; + + suffix = PyUnicode_FromString("])"); + if (suffix == NULL) goto error; + } else { + prefix = PyUnicode_FromString(""); + if (prefix == NULL) goto error; + + mid = PyUnicode_FromString("["); + if (mid == NULL) goto error; + + suffix = PyUnicode_FromString("]"); + if (suffix == NULL) goto error; + } + + if (PyList_Append(l, prefix) < 0) goto error; + if (PyList_Append(l, mid) < 0) goto error; + + Py_ssize_t len = PyObject_Length(vec); + if (len < 0) goto error; + + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *it = PySequence_GetItem(vec, i); + if (it == NULL) goto error; + + PyObject *r; + if (depth == 0 || it == Py_None) { + r = PyObject_Repr(it); + } else { + r = Vec_GenericRepr(it, item_type, depth - 1, 0); + } + Py_DECREF(it); + + if (r == NULL) goto error; + + if (PyList_Append(l, r) < 0) { + Py_DECREF(r); + goto error; + } + Py_DECREF(r); + + if (i + 1 < len) { + if (PyList_Append(l, comma) < 0) goto error; + } + } + + if (PyList_Append(l, suffix) < 0) goto error; + + result = PyUnicode_Join(sep, l); + +error: + Py_XDECREF(l); + Py_XDECREF(prefix); + Py_XDECREF(mid); + Py_XDECREF(suffix); + Py_XDECREF(sep); + Py_XDECREF(comma); + return result; +} + +// Generic comparison implementation for vecs with PyObject * items. +PyObject *Vec_GenericRichcompare(Py_ssize_t *len, PyObject **items, + Py_ssize_t *other_len, PyObject **other_items, + int op) { + int cmp = 1; + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (*len != *other_len) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < *len && i < *other_len; i++) { + int itemcmp = PyObject_RichCompareBool(items[i], other_items[i], Py_EQ); + if (itemcmp < 0) + return NULL; + if (!itemcmp) { + cmp = 0; + break; + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + +int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item) { + for (Py_ssize_t i = 0; i < *len; i++) { + int match = 0; + if (items[i] == item) + match = 1; + else { + int itemcmp = PyObject_RichCompareBool(items[i], item, Py_EQ); + if (itemcmp < 0) + return 0; + match = itemcmp; + } + if (match) { + Py_CLEAR(items[i]); + for (; i < *len - 1; i++) { + items[i] = items[i + 1]; + } + (*len)--; + return 1; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + return 0; +} + +PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { + if (index < 0) + index += *len; + + if (index < 0 || index >= *len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + + PyObject *item = items[index]; + for (Py_ssize_t i = index; i < *len - 1; i++) + items[i] = items[i + 1]; + + (*len)--; + return item; +} + +PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; + + return Vec_GenericPop(len, items, index); +} + +// Module-level functions + +static PyObject *vec_append(PyObject *self, PyObject *args) +{ + PyObject *vec; + PyObject *item; + + if (!PyArg_ParseTuple(args, "OO", &vec, &item)) + return NULL; + + if (VecI64_Check(vec)) { + int64_t x = VecI64_UnboxItem(item); + if (VecI64_IsUnboxError(x)) { + return NULL; + } + VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); + v = VecI64_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI64_Box(v); + } else if (VecU8_Check(vec)) { + uint8_t x = VecU8_UnboxItem(item); + if (VecU8_IsUnboxError(x)) { + return NULL; + } + VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); + v = VecU8_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecU8_Box(v); + } else if (VecFloat_Check(vec)) { + double x = VecFloat_UnboxItem(item); + if (VecFloat_IsUnboxError(x)) { + return NULL; + } + VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); + v = VecFloat_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecFloat_Box(v); + } else if (VecI32_Check(vec)) { + int32_t x = VecI32_UnboxItem(item); + if (VecI32_IsUnboxError(x)) { + return NULL; + } + VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); + v = VecI32_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI32_Box(v); + } else if (VecI16_Check(vec)) { + int16_t x = VecI16_UnboxItem(item); + if (VecI16_IsUnboxError(x)) { + return NULL; + } + VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); + v = VecI16_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI16_Box(v); + } else if (VecBool_Check(vec)) { + double x = VecBool_UnboxItem(item); + if (VecBool_IsUnboxError(x)) { + return NULL; + } + VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); + v = VecBool_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecBool_Box(v); + } else { + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); + return NULL; + } +} + +static PyObject *vec_remove(PyObject *self, PyObject *args) +{ + PyObject *vec; + PyObject *item; + + if (!PyArg_ParseTuple(args, "OO", &vec, &item)) + return NULL; + + if (VecI64_Check(vec)) { + int64_t x = VecI64_UnboxItem(item); + if (VecI64_IsUnboxError(x)) { + return NULL; + } + VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); + v = VecI64_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI64_Box(v); + } else if (VecU8_Check(vec)) { + uint8_t x = VecU8_UnboxItem(item); + if (VecU8_IsUnboxError(x)) { + return NULL; + } + VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); + v = VecU8_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecU8_Box(v); + } else if (VecFloat_Check(vec)) { + double x = VecFloat_UnboxItem(item); + if (VecFloat_IsUnboxError(x)) { + return NULL; + } + VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); + v = VecFloat_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecFloat_Box(v); + } else if (VecI32_Check(vec)) { + int32_t x = VecI32_UnboxItem(item); + if (VecI32_IsUnboxError(x)) { + return NULL; + } + VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); + v = VecI32_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI32_Box(v); + } else if (VecI16_Check(vec)) { + int16_t x = VecI16_UnboxItem(item); + if (VecI16_IsUnboxError(x)) { + return NULL; + } + VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); + v = VecI16_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecI16_Box(v); + } else if (VecBool_Check(vec)) { + char x = VecBool_UnboxItem(item); + if (VecBool_IsUnboxError(x)) { + return NULL; + } + VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); + v = VecBool_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return VecBool_Box(v); + } else { + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); + return NULL; + } +} + +static PyObject *vec_pop(PyObject *self, PyObject *args) +{ + PyObject *vec; + Py_ssize_t index = -1; + + if (!PyArg_ParseTuple(args, "O|n:pop", &vec, &index)) + return NULL; + + PyObject *result_item0; + PyObject *result_item1; + + if (VecI64_Check(vec)) { + VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); + VecI64PopResult r; + r = VecI64_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecI64_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecI64_BoxItem(r.f1); + } else if (VecU8_Check(vec)) { + VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); + VecU8PopResult r; + r = VecU8_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecU8_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecU8_BoxItem(r.f1); + } else if (VecFloat_Check(vec)) { + VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); + VecFloatPopResult r; + r = VecFloat_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecFloat_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecFloat_BoxItem(r.f1); + } else if (VecI32_Check(vec)) { + VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); + VecI32PopResult r; + r = VecI32_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecI32_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecI32_BoxItem(r.f1); + } else if (VecI16_Check(vec)) { + VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); + VecI16PopResult r; + r = VecI16_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecI16_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecI16_BoxItem(r.f1); + } else if (VecBool_Check(vec)) { + VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); + VecBoolPopResult r; + r = VecBool_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecBool_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecBool_BoxItem(r.f1); + } else { + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); + return NULL; + } + + if (result_item1 == NULL) { + Py_DECREF(result_item0); + return NULL; + } + + PyObject *res = PyTuple_New(2); + if (res == NULL) { + Py_DECREF(result_item0); + Py_DECREF(result_item1); + return NULL; + } + + PyTuple_SET_ITEM(res, 0, result_item0); + PyTuple_SET_ITEM(res, 1, result_item1); + return res; +} + +static VecCapsule Capsule = { + &Vec_I64API, + &Vec_I32API, + &Vec_I16API, + &Vec_U8API, + &Vec_FloatAPI, + &Vec_BoolAPI, +}; + +#endif // MYPYC_EXPERIMENTAL + +static PyMethodDef VecsMethods[] = { +#ifdef MYPYC_EXPERIMENTAL + {"append", vec_append, METH_VARARGS, "Append a value to the end of a vec"}, + {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value from a vec"}, + {"pop", vec_pop, METH_VARARGS, "Remove and return vec item at index (default last)"}, +#endif + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static int +librt_vecs_module_exec(PyObject *m) +{ +#ifdef MYPYC_EXPERIMENTAL + PyObject *ext = PyImport_ImportModule("mypy_extensions"); + if (ext == NULL) { + return -1; + } + + LibRTVecs_I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); + if (LibRTVecs_I64TypeObj == NULL) { + return -1; + } + if (!PyType_Check(LibRTVecs_I64TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i64 is not a type"); + return -1; + } + LibRTVecs_I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); + if (LibRTVecs_I32TypeObj == NULL) { + return -1; + } + if (!PyType_Check(LibRTVecs_I32TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i32 is not a type"); + return -1; + } + LibRTVecs_I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); + if (LibRTVecs_I16TypeObj == NULL) { + return -1; + } + if (!PyType_Check(LibRTVecs_I16TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i16 is not a type"); + return -1; + } + LibRTVecs_U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); + if (LibRTVecs_U8TypeObj == NULL) { + return -1; + } + if (!PyType_Check(LibRTVecs_U8TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.u8 is not a type"); + return -1; + } + + if (PyType_Ready(&VecType) < 0) + return -1; + if (PyType_Ready(&VecGenericAliasType) < 0) + return -1; + + if (PyType_Ready(&VecI64Type) < 0) + return -1; + if (PyType_Ready(&VecI64BufType) < 0) + return -1; + if (PyType_Ready(&VecI32Type) < 0) + return -1; + if (PyType_Ready(&VecI32BufType) < 0) + return -1; + if (PyType_Ready(&VecI16Type) < 0) + return -1; + if (PyType_Ready(&VecI16BufType) < 0) + return -1; + if (PyType_Ready(&VecU8Type) < 0) + return -1; + if (PyType_Ready(&VecU8BufType) < 0) + return -1; + if (PyType_Ready(&VecFloatType) < 0) + return -1; + if (PyType_Ready(&VecFloatBufType) < 0) + return -1; + if (PyType_Ready(&VecBoolType) < 0) + return -1; + if (PyType_Ready(&VecBoolBufType) < 0) + return -1; + + Py_INCREF(&VecType); + if (PyModule_AddObject(m, "vec", (PyObject *)&VecType) < 0) { + Py_DECREF(&VecType); + return -1; + } + + PyObject *c_api = PyCapsule_New(&Capsule, "librt.vecs._C_API", NULL); + if (c_api == NULL) + return -1; + + if (PyModule_AddObject(m, "_C_API", c_api) < 0) { + Py_XDECREF(c_api); + Py_DECREF(&VecType); + return -1; + } + + Py_DECREF(ext); +#endif + return 0; +} + +static PyModuleDef_Slot librt_vecs_module_slots[] = { + {Py_mod_exec, librt_vecs_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; + +static PyModuleDef vecsmodule = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "vecs", + .m_doc = "This module implements the vec type and related functionality.", + .m_size = 0, + .m_methods = VecsMethods, + .m_slots = librt_vecs_module_slots, +}; + +PyMODINIT_FUNC +PyInit_vecs(void) +{ + return PyModuleDef_Init(&vecsmodule); +} diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h new file mode 100644 index 0000000000000..f29d696bac665 --- /dev/null +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -0,0 +1,630 @@ +#ifndef VEC_H_INCL +#define VEC_H_INCL + +// Header for the implementation of librt.vecs, which defines the 'vec' type. +// Refer to librt_vecs.c for more detailed information. + +#ifndef MYPYC_EXPERIMENTAL + +static int +import_librt_vecs(void) +{ + // All librt.vecs features are experimental for now, so don't set up the API here + return 0; +} + +#else // MYPYC_EXPERIMENTAL + +#define PY_SSIZE_T_CLEAN +#include +#include + +// Magic (native) integer return value on exception. Caller must also +// use PyErr_Occurred() since this overlaps with valid integer values. +#define MYPYC_INT_ERROR -113 + +// Item type constants for supported packed/specialized item types; must be +// even but not a multiple of 4 (2 + 4 * n). Each of these has a corresponding +// distinct implementation C extension class. For example, vec[i64] has a +// different runtime type than vec[i32]. All other item types use generic +// implementations. +#define VEC_ITEM_TYPE_I64 2 +#define VEC_ITEM_TYPE_I32 6 +#define VEC_ITEM_TYPE_I16 10 +#define VEC_ITEM_TYPE_U8 14 +#define VEC_ITEM_TYPE_FLOAT 18 +#define VEC_ITEM_TYPE_BOOL 22 + +static inline size_t Vec_IsMagicItemType(size_t item_type) { + return item_type & 2; +} + + +// Buffer objects + + +// vecbuf[i64] +typedef struct _VecI64BufObject { + PyObject_VAR_HEAD + int64_t items[1]; +} VecI64BufObject; + +// vecbuf[i32] +typedef struct _VecI32BufObject { + PyObject_VAR_HEAD + int32_t items[1]; +} VecI32BufObject; + +// vecbuf[i16] +typedef struct _VecI16BufObject { + PyObject_VAR_HEAD + int16_t items[1]; +} VecI16BufObject; + +// vecbuf[u8] +typedef struct _VecU8BufObject { + PyObject_VAR_HEAD + uint8_t items[1]; +} VecU8BufObject; + +// vecbuf[float] +typedef struct _VecFloatBufObject { + PyObject_VAR_HEAD + double items[1]; +} VecFloatBufObject; + +// vecbuf[bool] +typedef struct _VecBoolBufObject { + PyObject_VAR_HEAD + char items[1]; +} VecBoolBufObject; + +// Simple generic vecbuf: vecbuf[t] when t is a type object +typedef struct _VecTBufObject { + PyObject_VAR_HEAD + // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. + size_t item_type; + PyObject *items[1]; +} VecTBufObject; + +typedef struct _VecNestedBufItem { + Py_ssize_t len; + PyObject *buf; +} VecNestedBufItem; + +// Nested vec type: vec[vec[...]], vec[vec[...] | None], etc. +typedef struct _VecNestedBufObject { + PyObject_VAR_HEAD + // Tagged pointer to PyTypeObject *. Lowest bit is set for optional item type. + // The second lowest bit is set for a packed item type (VEC_ITEM_TYPE_*). + size_t item_type; + // Number of nested vec types (of any kind, at least 1) + size_t depth; + VecNestedBufItem items[1]; +} VecNestedBufObject; + + +// Unboxed vec objects + + +typedef struct _VecI64 { + Py_ssize_t len; + VecI64BufObject *buf; +} VecI64; + +typedef struct _VecI32 { + Py_ssize_t len; + VecI32BufObject *buf; +} VecI32; + +typedef struct _VecI16 { + Py_ssize_t len; + VecI16BufObject *buf; +} VecI16; + +typedef struct _VecU8 { + Py_ssize_t len; + VecU8BufObject *buf; +} VecU8; + +typedef struct _VecFloat { + Py_ssize_t len; + VecFloatBufObject *buf; +} VecFloat; + +typedef struct _VecBool { + Py_ssize_t len; + VecBoolBufObject *buf; +} VecBool; + +typedef struct _VecT { + Py_ssize_t len; + VecTBufObject *buf; +} VecT; + +typedef struct _VecNested { + Py_ssize_t len; + VecNestedBufObject *buf; +} VecNested; + + +// Boxed vec objects + + +// Arbitrary boxed vec object (only shared bits) +typedef struct _VecObject { + PyObject_HEAD + Py_ssize_t len; +} VecObject; + +// Boxed vec[i64] +typedef struct _VecI64Object { + PyObject_HEAD + VecI64 vec; +} VecI64Object; + +// Boxed vec[i32] +typedef struct _VecI32Object { + PyObject_HEAD + VecI32 vec; +} VecI32Object; + +// Boxed vec[i16] +typedef struct _VecI16Object { + PyObject_HEAD + VecI16 vec; +} VecI16Object; + +// Boxed vec[u8] +typedef struct _VecU8Object { + PyObject_HEAD + VecU8 vec; +} VecU8Object; + +// Boxed vec[float] +typedef struct _VecFloatObject { + PyObject_HEAD + VecFloat vec; +} VecFloatObject; + +// Boxed vec[bool] +typedef struct _VecBoolObject { + PyObject_HEAD + VecBool vec; +} VecBoolObject; + +// Simple boxed generic vecbuf: vecbuf[t] when t is a type object +typedef struct _VecTObject { + PyObject_HEAD + VecT vec; +} VecTObject; + +// Extended generic vec type: vec[t | None], vec[vec[...]], etc. +typedef struct _VecNestedObject { + PyObject_HEAD + VecNested vec; +} VecNestedObject; + + +#ifndef MYPYC_DECLARED_tuple_T2V88 +#define MYPYC_DECLARED_tuple_T2V88 +typedef struct tuple_T2V88 { + VecI64 f0; + int64_t f1; +} tuple_T2V88; +static tuple_T2V88 tuple_undefined_T2V88 = { { -1, NULL } , 0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2V44 +#define MYPYC_DECLARED_tuple_T2V44 +typedef struct tuple_T2V44 { + VecI32 f0; + int32_t f1; +} tuple_T2V44; +static tuple_T2V44 tuple_undefined_T2V44 = { { -1, NULL } , 0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2V22 +#define MYPYC_DECLARED_tuple_T2V22 +typedef struct tuple_T2V22 { + VecI16 f0; + int16_t f1; +} tuple_T2V22; +static tuple_T2V22 tuple_undefined_T2V22 = { { -1, NULL } , 0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2VU1U1 +#define MYPYC_DECLARED_tuple_T2VU1U1 +typedef struct tuple_T2VU1U1 { + VecU8 f0; + uint8_t f1; +} tuple_T2VU1U1; +static tuple_T2VU1U1 tuple_undefined_T2VU1U1 = { { -1, NULL } , 0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2VFF +#define MYPYC_DECLARED_tuple_T2VFF +typedef struct tuple_T2VFF { + VecFloat f0; + double f1; +} tuple_T2VFF; +static tuple_T2VFF tuple_undefined_T2VFF = { { -1, NULL } , 0.0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2VCC +#define MYPYC_DECLARED_tuple_T2VCC +typedef struct tuple_T2VCC { + VecBool f0; + char f1; +} tuple_T2VCC; +static tuple_T2VCC tuple_undefined_T2VCC = { { -1, NULL } , 0 }; +#endif + +typedef tuple_T2V88 VecI64PopResult; +typedef tuple_T2V44 VecI32PopResult; +typedef tuple_T2V22 VecI16PopResult; +typedef tuple_T2VU1U1 VecU8PopResult; +typedef tuple_T2VFF VecFloatPopResult; +typedef tuple_T2VCC VecBoolPopResult; + +// vec[i64] operations + type objects (stored in a capsule) +typedef struct _VecI64API { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI64 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecI64); + VecI64 (*unbox)(PyObject *); + VecI64 (*convert_from_nested)(VecNestedBufItem); + VecI64 (*append)(VecI64, int64_t); + VecI64PopResult (*pop)(VecI64, Py_ssize_t); + VecI64 (*remove)(VecI64, int64_t); + // TODO: Py_ssize_t + VecI64 (*slice)(VecI64, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, int64_t); + // iter? +} VecI64API; + +// vec[i32] operations + type objects (stored in a capsule) +typedef struct _VecI32API { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI32 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecI32); + VecI32 (*unbox)(PyObject *); + VecI32 (*convert_from_nested)(VecNestedBufItem); + VecI32 (*append)(VecI32, int32_t); + VecI32PopResult (*pop)(VecI32, Py_ssize_t); + VecI32 (*remove)(VecI32, int32_t); + // TODO: Py_ssize_t + VecI32 (*slice)(VecI32, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, int32_t); + // iter? +} VecI32API; + +// vec[i16] operations + type objects (stored in a capsule) +typedef struct _VecI16API { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI16 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecI16); + VecI16 (*unbox)(PyObject *); + VecI16 (*convert_from_nested)(VecNestedBufItem); + VecI16 (*append)(VecI16, int16_t); + VecI16PopResult (*pop)(VecI16, Py_ssize_t); + VecI16 (*remove)(VecI16, int16_t); + // TODO: Py_ssize_t + VecI16 (*slice)(VecI16, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, int16_t); + // iter? +} VecI16API; + +// vec[u8] operations + type objects (stored in a capsule) +typedef struct _VecU8API { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecU8 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecU8); + VecU8 (*unbox)(PyObject *); + VecU8 (*convert_from_nested)(VecNestedBufItem); + VecU8 (*append)(VecU8, uint8_t); + VecU8PopResult (*pop)(VecU8, Py_ssize_t); + VecU8 (*remove)(VecU8, uint8_t); + // TODO: Py_ssize_t + VecU8 (*slice)(VecU8, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, uint8_t); + // iter? +} VecU8API; + +// vec[float] operations + type objects (stored in a capsule) +typedef struct _VecFloatAPI { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecFloat (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecFloat); + VecFloat (*unbox)(PyObject *); + VecFloat (*convert_from_nested)(VecNestedBufItem); + VecFloat (*append)(VecFloat, double); + VecFloatPopResult (*pop)(VecFloat, Py_ssize_t); + VecFloat (*remove)(VecFloat, double); + // TODO: Py_ssize_t + VecFloat (*slice)(VecFloat, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, double); + // iter? +} VecFloatAPI; + +// vec[bool] operations + type objects (stored in a capsule) +typedef struct _VecBoolAPI { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecBool (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecBool); + VecBool (*unbox)(PyObject *); + VecBool (*convert_from_nested)(VecNestedBufItem); + VecBool (*append)(VecBool, char); + VecBoolPopResult (*pop)(VecBool, Py_ssize_t); + VecBool (*remove)(VecBool, char); + // TODO: Py_ssize_t + VecBool (*slice)(VecBool, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, char); + // iter? +} VecBoolAPI; + +typedef struct { + VecI64API *i64; + VecI32API *i32; + VecI16API *i16; + VecU8API *u8; + VecFloatAPI *float_; + VecBoolAPI *bool_; +} VecCapsule; + +#define VEC_BUF_SIZE(b) ((b)->ob_base.ob_size) +#define VEC_ITEM_TYPE(t) ((PyTypeObject *)((t) & ~1)) +#define VEC_BUF_ITEM_TYPE(b) VEC_ITEM_TYPE((b)->item_type) +#define VEC_CAP(v) ((v).buf->ob_base.ob_size) +#define VEC_IS_ERROR(v) ((v).len < 0) +#define VEC_DECREF(v) Py_XDECREF((v).buf) +#define VEC_INCREF(v) Py_XINCREF((v).buf) + +// Type objects + +// Buffer type objects that store vec items +extern PyTypeObject VecI64BufType; +extern PyTypeObject VecI32BufType; +extern PyTypeObject VecI16BufType; +extern PyTypeObject VecU8BufType; +extern PyTypeObject VecFloatBufType; +extern PyTypeObject VecBoolBufType; + +// Wrapper type objects for boxed vec values +extern PyTypeObject VecI64Type; +extern PyTypeObject VecI32Type; +extern PyTypeObject VecI16Type; +extern PyTypeObject VecU8Type; +extern PyTypeObject VecFloatType; +extern PyTypeObject VecBoolType; + +// Type objects corresponding to the 'i64', 'i32', 'i16, and 'u8' types +extern PyTypeObject *LibRTVecs_I64TypeObj; +extern PyTypeObject *LibRTVecs_I32TypeObj; +extern PyTypeObject *LibRTVecs_I16TypeObj; +extern PyTypeObject *LibRTVecs_U8TypeObj; + +extern VecI64API Vec_I64API; +extern VecI32API Vec_I32API; +extern VecI16API Vec_I16API; +extern VecU8API Vec_U8API; +extern VecFloatAPI Vec_FloatAPI; +extern VecBoolAPI Vec_BoolAPI; + +static inline int Vec_CheckFloatError(PyObject *o) { + if (PyFloat_Check(o)) { + PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); + return 1; + } + return 0; +} + +// vec[i64] operations + +static inline int VecI64_Check(PyObject *o) { + return o->ob_type == &VecI64Type; +} + +static inline PyObject *VecI64_BoxItem(int64_t x) { + return PyLong_FromLongLong(x); +} + +static inline int64_t VecI64_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + return PyLong_AsLongLong(o); +} + +static inline int VecI64_IsUnboxError(int64_t x) { + return x == -1 && PyErr_Occurred(); +} + +PyObject *VecI64_Box(VecI64); +VecI64 VecI64_Append(VecI64, int64_t x); +VecI64 VecI64_Remove(VecI64, int64_t x); +VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); + +// vec[i32] operations + +static inline int VecI32_Check(PyObject *o) { + return o->ob_type == &VecI32Type; +} + +static inline PyObject *VecI32_BoxItem(int32_t x) { + return PyLong_FromLongLong(x); +} + +static inline int32_t VecI32_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + long x = PyLong_AsLong(o); + if (x > INT32_MAX || x < INT32_MIN) { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to i32"); + return -1; + } + return x; +} + +static inline int VecI32_IsUnboxError(int32_t x) { + return x == -1 && PyErr_Occurred(); +} + +PyObject *VecI32_Box(VecI32); +VecI32 VecI32_Append(VecI32, int32_t x); +VecI32 VecI32_Remove(VecI32, int32_t x); +VecI32PopResult VecI32_Pop(VecI32 v, Py_ssize_t index); + +// vec[i16] operations + +static inline int VecI16_Check(PyObject *o) { + return o->ob_type == &VecI16Type; +} + +static inline PyObject *VecI16_BoxItem(int16_t x) { + return PyLong_FromLongLong(x); +} + +static inline int16_t VecI16_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + long x = PyLong_AsLong(o); + if (x >= 32768 || x < -32768) { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to i16"); + return -1; + } + return x; +} + +static inline int VecI16_IsUnboxError(int16_t x) { + return x == -1 && PyErr_Occurred(); +} + +PyObject *VecI16_Box(VecI16); +VecI16 VecI16_Append(VecI16, int16_t x); +VecI16 VecI16_Remove(VecI16, int16_t x); +VecI16PopResult VecI16_Pop(VecI16 v, Py_ssize_t index); + +// vec[u8] operations + +static inline int VecU8_Check(PyObject *o) { + return o->ob_type == &VecU8Type; +} + +static inline PyObject *VecU8_BoxItem(uint8_t x) { + return PyLong_FromUnsignedLong(x); +} + +static inline uint8_t VecU8_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + unsigned long x = PyLong_AsUnsignedLong(o); + if (x <= 255) + return x; + else if (x == (unsigned long)-1) + return 239; + else { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to u8"); + return 239; + } +} + +static inline int VecU8_IsUnboxError(uint8_t x) { + return x == 239 && PyErr_Occurred(); +} + +PyObject *VecU8_Box(VecU8); +VecU8 VecU8_Append(VecU8, uint8_t x); +VecU8 VecU8_Remove(VecU8, uint8_t x); +VecU8PopResult VecU8_Pop(VecU8 v, Py_ssize_t index); + +// vec[float] operations + +static inline int VecFloat_Check(PyObject *o) { + return o->ob_type == &VecFloatType; +} + +static inline PyObject *VecFloat_BoxItem(double x) { + return PyFloat_FromDouble(x); +} + +static inline double VecFloat_UnboxItem(PyObject *o) { + return PyFloat_AsDouble(o); +} + +static inline int VecFloat_IsUnboxError(double x) { + return x == -1.0 && PyErr_Occurred(); +} + +PyObject *VecFloat_Box(VecFloat); +VecFloat VecFloat_Append(VecFloat, double x); +VecFloat VecFloat_Remove(VecFloat, double x); +VecFloatPopResult VecFloat_Pop(VecFloat v, Py_ssize_t index); + +// vec[bool] operations + +static inline int VecBool_Check(PyObject *o) { + return o->ob_type == &VecBoolType; +} + +static inline PyObject *VecBool_BoxItem(char x) { + if (x == 1) { + Py_INCREF(Py_True); + return Py_True; + } else { + Py_INCREF(Py_False); + return Py_False; + } +} + +static inline char VecBool_UnboxItem(PyObject *o) { + if (o == Py_False) { + return 0; + } else if (o == Py_True) { + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "bool value expected"); + return 2; + } +} + +static inline int VecBool_IsUnboxError(char x) { + return x == 2; +} + +PyObject *VecBool_Box(VecBool); +VecBool VecBool_Append(VecBool, char x); +VecBool VecBool_Remove(VecBool, char x); +VecBoolPopResult VecBool_Pop(VecBool v, Py_ssize_t index); + +// Misc helpers + +PyObject *Vec_TypeToStr(size_t item_type, size_t depth); +PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose); +PyObject *Vec_GenericRichcompare(Py_ssize_t *len, PyObject **items, + Py_ssize_t *other_len, PyObject **other_items, + int op); +int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item); +PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args); +PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); + +#endif // MYPYC_EXPERIMENTAL + +#endif // VEC_H_INCL diff --git a/mypyc/lib-rt/vecs/vec_bool.c b/mypyc/lib-rt/vecs/vec_bool.c new file mode 100644 index 0000000000000..ea324164779c9 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_bool.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecBool +#define VEC_TYPE VecBoolType +#define VEC_OBJECT VecBoolObject +#define BUF_OBJECT VecBoolBufObject +#define BUF_TYPE VecBoolBufType +#define NAME(suffix) VecBool##suffix +#define FUNC(suffix) VecBool_##suffix +#define ITEM_TYPE_STR "bool" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_BOOL +#define ITEM_C_TYPE char +#define FEATURES Vec_BoolAPI + +#define BOX_ITEM VecBool_BoxItem +#define UNBOX_ITEM VecBool_UnboxItem +#define IS_UNBOX_ERROR VecBool_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c new file mode 100644 index 0000000000000..3952c01e711cb --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecFloat +#define VEC_TYPE VecFloatType +#define VEC_OBJECT VecFloatObject +#define BUF_OBJECT VecFloatBufObject +#define BUF_TYPE VecFloatBufType +#define NAME(suffix) VecFloat##suffix +#define FUNC(suffix) VecFloat_##suffix +#define ITEM_TYPE_STR "float" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_FLOAT +#define ITEM_C_TYPE double +#define FEATURES Vec_FloatAPI + +#define BOX_ITEM VecFloat_BoxItem +#define UNBOX_ITEM VecFloat_UnboxItem +#define IS_UNBOX_ERROR VecFloat_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i16.c b/mypyc/lib-rt/vecs/vec_i16.c new file mode 100644 index 0000000000000..85a0055225d56 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i16.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecI16 +#define VEC_TYPE VecI16Type +#define VEC_OBJECT VecI16Object +#define BUF_OBJECT VecI16BufObject +#define BUF_TYPE VecI16BufType +#define NAME(suffix) VecI16##suffix +#define FUNC(suffix) VecI16_##suffix +#define ITEM_TYPE_STR "i16" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I16 +#define ITEM_C_TYPE int16_t +#define FEATURES Vec_I16API + +#define BOX_ITEM VecI16_BoxItem +#define UNBOX_ITEM VecI16_UnboxItem +#define IS_UNBOX_ERROR VecI16_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i32.c b/mypyc/lib-rt/vecs/vec_i32.c new file mode 100644 index 0000000000000..9eb595994483a --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i32.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecI32 +#define VEC_TYPE VecI32Type +#define VEC_OBJECT VecI32Object +#define BUF_OBJECT VecI32BufObject +#define BUF_TYPE VecI32BufType +#define NAME(suffix) VecI32##suffix +#define FUNC(suffix) VecI32_##suffix +#define ITEM_TYPE_STR "i32" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I32 +#define ITEM_C_TYPE int32_t +#define FEATURES Vec_I32API + +#define BOX_ITEM VecI32_BoxItem +#define UNBOX_ITEM VecI32_UnboxItem +#define IS_UNBOX_ERROR VecI32_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c new file mode 100644 index 0000000000000..e6b7df693333f --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecI64 +#define VEC_TYPE VecI64Type +#define VEC_OBJECT VecI64Object +#define BUF_OBJECT VecI64BufObject +#define BUF_TYPE VecI64BufType +#define NAME(suffix) VecI64##suffix +#define FUNC(suffix) VecI64_##suffix +#define ITEM_TYPE_STR "i64" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I64 +#define ITEM_C_TYPE int64_t +#define FEATURES Vec_I64API + +#define BOX_ITEM VecI64_BoxItem +#define UNBOX_ITEM VecI64_UnboxItem +#define IS_UNBOX_ERROR VecI64_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c new file mode 100644 index 0000000000000..864074cd10878 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -0,0 +1,397 @@ +#ifdef MYPYC_EXPERIMENTAL +// NOTE: This file can't be compiled on its own, it must be #included +// with certain #defines set, as described below. +// +// Implementation of a vec class specialized to a specific item type, such +// as vec[i64] or vec[float]. Assume that certain #defines are provided that +// provide all the item type specific definitions: +// +// VEC vec C type (e.g. VecI32) +// VEC_TYPE boxed vec type object (e.g. VecI32Type) +// VEC_OBJECT boxed Python object struct (e.g. VecI32Object) +// BUF_OBJECT buffer Python object struct (e.g. VecI32BufObject) +// BUF_TYPE buffer type object (e.g. VecI32BufType) +// NAME(suffix) macro to create prefixed name with given suffix (e.g. VecI32##suffix) +// FUNC(suffix) macro to create prefixed function name with suffix (e.g. VecI32_##suffix) +// ITEM_TYPE_STR vec item type as C string literal (e.g. "i32") +// ITEM_TYPE_MAGIC integer constant corresponding to the item type (e.g. VEC_ITEM_TYPE_I32) +// ITEM_C_TYPE C type used for items (e.g. int32_t) +// FEATURES capsule API struct name (e.g. Vec_I32API) +// BOX_ITEM C function to box item (e.g. VecI32_BoxItem) +// UNBOX_ITEM C function to unbox item (e.g. VecI32_UnboxItem) +// IS_UNBOX_ERROR C function to check for unbox error (e.g. VecI32_IsUnboxError) + +#ifndef VEC +#error "VEC must be defined" +#endif + +#define PY_SSIZE_T_CLEAN +#include +#include "librt_vecs.h" + +inline static VEC vec_error() { + VEC v = { .len = -1 }; + return v; +} + +// Alloc a partially initialized vec. Caller *must* initialize len. +static VEC vec_alloc(Py_ssize_t size) +{ + BUF_OBJECT *buf; + /* TODO: Check for overflow */ + if (size == 0) { + buf = NULL; + } else { + buf = PyObject_NewVar(BUF_OBJECT, &BUF_TYPE, size); + if (buf == NULL) + return vec_error(); + } + VEC res = { .buf = buf }; + return res; +} + +static void vec_dealloc(VEC_OBJECT *self) { + Py_CLEAR(self->vec.buf); + PyObject_Del(self); +} + +// Box a vec[] value, stealing 'vec'. On error, decref 'vec'. +PyObject *FUNC(Box)(VEC vec) { + VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); + if (obj == NULL) { + VEC_DECREF(vec); + return NULL; + } + obj->vec = vec; + return (PyObject *)obj; +} + +VEC FUNC(Unbox)(PyObject *obj) { + if (obj->ob_type == &VEC_TYPE) { + VEC result = ((VEC_OBJECT *)obj)->vec; + VEC_INCREF(result); // TODO: Should we borrow instead? + return result; + } else { + PyErr_SetString(PyExc_TypeError, "vec[" ITEM_TYPE_STR "] expected"); + return vec_error(); + } +} + +VEC FUNC(ConvertFromNested)(VecNestedBufItem item) { + return (VEC) { item.len, (BUF_OBJECT *)item.buf }; +} + +VEC FUNC(New)(Py_ssize_t size, Py_ssize_t cap) { + if (cap < size) + size = cap; + VEC vec = vec_alloc(cap); + if (VEC_IS_ERROR(vec)) + return vec; + for (Py_ssize_t i = 0; i < cap; i++) { + vec.buf->items[i] = 0; + } + vec.len = size; + return vec; +} + +PyObject *FUNC(FromIterable)(PyObject *iterable) { + VEC v = vec_alloc(0); + if (VEC_IS_ERROR(v)) + return NULL; + v.len = 0; + + PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + VEC_DECREF(v); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + ITEM_C_TYPE x = UNBOX_ITEM(item); + Py_DECREF(item); + if (IS_UNBOX_ERROR(x)) { + Py_DECREF(iter); + VEC_DECREF(v); + return NULL; + } + v = FUNC(Append)(v, x); + if (VEC_IS_ERROR(v)) { + Py_DECREF(iter); + VEC_DECREF(v); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) { + VEC_DECREF(v); + return NULL; + } + return FUNC(Box)(v); +} + +static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { + static char *kwlist[] = {"", NULL}; + PyObject *init = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { + return NULL; + } + if (init == NULL) { + return FUNC(Box)(FUNC(New)(0, 0)); + } else { + return (PyObject *)FUNC(FromIterable)(init); + } +} + +static PyObject *vec_repr(PyObject *self) { + return Vec_GenericRepr(self, ITEM_TYPE_MAGIC, 0, 1); +} + +static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { + VEC v = ((VEC_OBJECT *)o)->vec; + if ((size_t)i < (size_t)v.len) { + return BOX_ITEM(v.buf->items[i]); + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + return BOX_ITEM(v.buf->items[i + v.len]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +VEC FUNC(Slice)(VEC vec, int64_t start, int64_t end) { + if (start < 0) + start += vec.len; + if (end < 0) + end += vec.len; + if (start < 0) + start = 0; + if (start >= vec.len) + start = vec.len; + if (end < start) + end = start; + if (end > vec.len) + end = vec.len; + int64_t slicelength = end - start; + VEC res = vec_alloc(slicelength); + if (VEC_IS_ERROR(res)) + return res; + res.len = slicelength; + for (Py_ssize_t i = 0; i < slicelength; i++) + res.buf->items[i] = vec.buf->items[start + i]; + return res; +} + +static PyObject *vec_subscript(PyObject *self, PyObject *item) { + VEC vec = ((VEC_OBJECT *)self)->vec; + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if ((size_t)i < (size_t)vec.len) { + return BOX_ITEM(vec.buf->items[i]); + } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { + return BOX_ITEM(vec.buf->items[i + vec.len]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + } else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) + return NULL; + Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); + VEC res = vec_alloc(slicelength); + if (VEC_IS_ERROR(res)) + return NULL; + res.len = slicelength; + Py_ssize_t j = start; + for (Py_ssize_t i = 0; i < slicelength; i++) { + res.buf->items[i] = vec.buf->items[j]; + j += step; + } + return FUNC(Box)(res); + } else { + PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", + item->ob_type->tp_name); + return NULL; + } +} + +static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + ITEM_C_TYPE x = UNBOX_ITEM(o); + if (IS_UNBOX_ERROR(x)) + return -1; + VEC v = ((VEC_OBJECT *)self)->vec; + if ((size_t)i < (size_t)v.len) { + v.buf->items[i] = x; + return 0; + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + v.buf->items[i + v.len] = x; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + +static Py_ssize_t vec_length(PyObject *o) { + return ((VEC_OBJECT *)o)->vec.len; +} + +static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { + int cmp = 1; + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (other->ob_type != &VEC_TYPE) + cmp = 0; + else { + VEC x = ((VEC_OBJECT *)self)->vec; + VEC y = ((VEC_OBJECT *)other)->vec; + if (x.len != y.len) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < x.len; i++) { + if (x.buf->items[i] != y.buf->items[i]) { + cmp = 0; + break; + } + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + +// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. +VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { + if (vec.buf && vec.len < VEC_CAP(vec)) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; + } else { + Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; + Py_ssize_t new_size = 2 * cap + 1; + VEC new = vec_alloc(new_size); + if (VEC_IS_ERROR(new)) { + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(vec); + return vec_error(); + } + new.len = vec.len + 1; + if (vec.len > 0) + memcpy(new.buf->items, vec.buf->items, sizeof(ITEM_C_TYPE) * vec.len); + new.buf->items[vec.len] = x; + Py_XDECREF(vec.buf); + return new; + } +} + +// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. +VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { + for (Py_ssize_t i = 0; i < v.len; i++) { + if (v.buf->items[i] == x) { + for (; i < v.len - 1; i++) { + v.buf->items[i] = v.buf->items[i + 1]; + } + v.len--; + // Return the stolen reference without INCREF + return v; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); + return vec_error(); +} + +// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. +NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { + NAME(PopResult) result; + + if (index < 0) + index += v.len; + + if (index < 0 || index >= v.len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); + result.f0 = vec_error(); + result.f1 = 0; + return result; + } + + result.f1 = v.buf->items[index]; + for (Py_ssize_t i = index; i < v.len - 1; i++) { + v.buf->items[i] = v.buf->items[i + 1]; + } + + v.len--; + // Return the stolen reference without INCREF + result.f0 = v; + return result; +} + +static PyMappingMethods vec_mapping_methods = { + .mp_length = vec_length, + .mp_subscript = vec_subscript, +}; + +static PySequenceMethods vec_sequence_methods = { + .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, +}; + +static PyMethodDef vec_methods[] = { + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + +PyTypeObject BUF_TYPE = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecbuf[" ITEM_TYPE_STR "]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(BUF_OBJECT) - sizeof(ITEM_C_TYPE), + .tp_itemsize = sizeof(ITEM_C_TYPE), + .tp_flags = Py_TPFLAGS_DEFAULT, + //.tp_new = ?? + .tp_free = PyObject_Del, +}; + +PyTypeObject VEC_TYPE = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec[" ITEM_TYPE_STR "]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VEC_OBJECT), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vec_new, + //.tp_free = PyObject_Del, + .tp_dealloc = (destructor)vec_dealloc, + .tp_repr = (reprfunc)vec_repr, + .tp_as_sequence = &vec_sequence_methods, + .tp_as_mapping = &vec_mapping_methods, + .tp_richcompare = vec_richcompare, + .tp_methods = vec_methods, +}; + +NAME(API) FEATURES = { + &VEC_TYPE, + &BUF_TYPE, + FUNC(New), + FUNC(Box), + FUNC(Unbox), + FUNC(ConvertFromNested), + FUNC(Append), + FUNC(Pop), + FUNC(Remove), + FUNC(Slice), +}; + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_u8.c b/mypyc/lib-rt/vecs/vec_u8.c new file mode 100644 index 0000000000000..1d59f177c3104 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_u8.c @@ -0,0 +1,20 @@ +#ifdef MYPYC_EXPERIMENTAL +#define VEC VecU8 +#define VEC_TYPE VecU8Type +#define VEC_OBJECT VecU8Object +#define BUF_OBJECT VecU8BufObject +#define BUF_TYPE VecU8BufType +#define NAME(suffix) VecU8##suffix +#define FUNC(suffix) VecU8_##suffix +#define ITEM_TYPE_STR "u8" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_U8 +#define ITEM_C_TYPE uint8_t +#define FEATURES Vec_U8API + +#define BOX_ITEM VecU8_BoxItem +#define UNBOX_ITEM VecU8_UnboxItem +#define IS_UNBOX_ERROR VecU8_IsUnboxError + +#include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index b2700f731ddd6..60e99256c5671 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from collections.abc import Iterator import math +import sys from typing import ( Any, Iterator, TypeVar, Generator, Optional, List, Tuple, Sequence, Union, Callable, Awaitable, Generic @@ -101,3 +102,11 @@ def make_python_function(f: F) -> F: def g(*args: Any, **kwargs: Any) -> Any: return f(*args, **kwargs) return g # type: ignore + + +def is_gil_disabled() -> bool: + return hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() + + +def is_64_bit_platform() -> bool: + return getattr(sys, "maxsize") > 2**32 diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test new file mode 100644 index 0000000000000..a1dcde6219c1e --- /dev/null +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -0,0 +1,343 @@ +[case testLibrtVecsI64Interpreted_librt_experimental] +# Test cases for vec[i64], using generic operations (simulates use from interpreted code). +# +# These also act as test cases for specialized vec types in general (e.g. vec[float]), +# since much of the implementation is shared between all of them. +# +# There is also run-vecs-misc-interp.test with limited test coverage for vec[float] etc. + +import sys +from typing import Any +import typing + +import librt.vecs +import mypy_extensions + +from testutil import assertRaises, is_gil_disabled, is_64_bit_platform + +# Access via Any references to force generic operations +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") +i64: Any = mypy_extensions.i64 + +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + +def test_construct_from_initializer() -> None: + v = vec[i64]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i64](x + 1 for x in [5, 7, -8, 10]) + assert len(v) == 4 + assert v[0] == 6 + assert v[1] == 8 + assert v[2] == -7 + assert v[3] == 11 + with assertRaises(TypeError): + vec[i64](1) + with assertRaises(TypeError): + vec[i64](['1']) + with assertRaises(TypeError): + vec[i64]([None]) + +def test_append() -> None: + v = vec[i64]() + v = append(v, 1) + assert str(v) == "vec[i64]([1])" + for n in range(2, 10): + v = append(v, n) + assert str(v) == "vec[i64]([1, 2, 3, 4, 5, 6, 7, 8, 9])" + v = vec[i64]() + v = append(v, 2**63 - 1) + v = append(v, -2**63) + assert v[0] == 2**63 - 1 + assert v[1] == -2**63 + v = vec[i64]() + for i in range(10000): + v = append(v, i) + for i in range(10000): + assert v[i] == i + +def test_append_error() -> None: + v: Any = vec[i64]() + with assertRaises(OverflowError): + v = append(v, 2**63) + with assertRaises(OverflowError): + v = append(v, -2**63 - 1) + assert len(v) == 0 + with assertRaises(TypeError): + v = append(v, 'x') + with assertRaises(TypeError): + v = append(v, 1.5) + assert len(v) == 0 + +def test_repr() -> None: + assert str(vec[i64]()) == "vec[i64]([])" + assert str(vec[i64]([123])) == "vec[i64]([123])" + assert str(vec[i64]([123, -987654321])) == "vec[i64]([123, -987654321])" + +def test_size() -> None: + v = vec[i64]() + # Only test on 64-bit, non-free-threading builds + if not is_gil_disabled() and is_64_bit_platform(): + assert getsizeof(v) == 32 + v = append(v, 123) + assert getsizeof(v) == 32 + +def test_get_item() -> None: + v = append(vec[i64](), 44) + assert v[0] == 44 + v = append(v, 56) + assert v[0] == 44 + assert v[1] == 56 + +def test_get_item_error() -> None: + v = append(vec[i64](), 1) + v = append(v, 2) + v = append(v, 3) + with assertRaises(IndexError): + v[3] + with assertRaises(IndexError): + v[-4] + n = 2 ** 100 + with assertRaises(IndexError): + v[n] + +def test_get_item_negative() -> None: + v = vec[i64]([2, 5, 7]) + assert v[-1] == 7 + assert v[-2] == 5 + assert v[-3] == 2 + with assertRaises(IndexError): + v[-4] + with assertRaises(IndexError): + v[-2**62] + +def test_len() -> None: + v = vec[i64]() + assert len(v) == 0 + v = append(v, 1) + assert len(v) == 1 + v = append(v, 2) + assert len(v) == 2 + for i in range(100): + v = append(v, i) + assert len(v) == i + 3 + +def test_set_item() -> None: + v = vec[i64]() + v = append(v, 1) + v = append(v, 2) + v[0] = 3 + assert v[0] == 3 + assert v[1] == 2 + v[1] = 5 + assert v[0] == 3 + assert v[1] == 5 + +def test_set_item_error() -> None: + v = append(vec[i64](), 1) + v = append(v, 2) + v = append(v, 3) + with assertRaises(IndexError): + v[3] = 1 + v[0] = 2**63 - 1 + assert v[0] == 2**63 - 1 + with assertRaises(OverflowError): + v[0] = 2**63 + v[0] = -2**63 + assert v[0] == -2**63 + with assertRaises(OverflowError): + v[0] = -2**63 - 1 + with assertRaises(TypeError): + v[0] = "x" + with assertRaises(TypeError): + v[0] = 1.5 + +def test_set_item_negative() -> None: + v = vec[i64]() + v = append(v, 1) + v = append(v, 2) + v[-2] = 3 + assert v[0] == 3 + assert v[1] == 2 + v[-1] = 5 + assert v[0] == 3 + assert v[1] == 5 + with assertRaises(IndexError): + v[-3] = 1 + +def test_equality() -> None: + assert_eq(vec[i64](), vec[i64]()) + assert_eq(vec[i64]([1]), vec[i64]([1])) + v = vec[i64]([1]) + assert_eq(v, v) + assert_eq(vec[i64]([1, 2]), vec[i64]([1, 2])) + + assert_neq(vec[i64](), vec[i64]([1])) + assert_neq(vec[i64]([1, 2]), vec[i64]([1])) + assert_neq(vec[i64]([1]), vec[i64]([2])) + assert_neq(vec[i64]([1, 2]), vec[i64]([2, 2])) + assert_neq(vec[i64]([1, 2]), vec[i64]([1, 1])) + +def test_equality_different_types() -> None: + assert_neq(vec[i64]([1]), None) + assert_neq(vec[i64]([1]), [1]) + assert_neq(vec[i64]([1]), (1,)) + +def assert_eq(x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b + +def assert_neq(x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + +def test_iteration() -> None: + for x in vec[i64](): + assert False + a = [] + for x in vec[i64]([11]): + a.append(x) + assert a == [11] + a = [] + for x in vec[i64]([11, 23, 45, 64]): + a.append(x) + assert a == [11, 23, 45, 64] + +def test_iteration_nested() -> None: + v = vec[i64]() + v = append(v, 2) + v = append(v, 5) + v = append(v, 7) + a = [] + for x in v: + for y in v: + a.append((x, y)) + assert a == [(2, 2), (2, 5), (2, 7), + (5, 2), (5, 5), (5, 7), + (7, 2), (7, 5), (7, 7)] + +def test_slicing() -> None: + v = vec[i64](range(5)) + assert v[1:4] == vec[i64]([1, 2, 3]) + assert v[2:-1] == vec[i64]([2, 3]) + assert v[-2:-1] == vec[i64]([3]) + assert v[1:] == vec[i64]([1, 2, 3, 4]) + assert v[:-1] == vec[i64]([0, 1, 2, 3]) + assert v[:] == v + assert v[:] is not v + assert v[5:] == vec[i64]() + assert v[100:] == vec[i64]() + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[i64]([2, 3, 4]) + assert v[-100:2] == vec[i64]([0, 1]) + assert v[5:100] == vec[i64]([]) + assert v[50:100] == vec[i64]([]) + assert v[-100:-50] == vec[i64]([]) + +def test_slicing_with_step() -> None: + v = vec[i64](range(10)) + assert v[1:5:2] == vec[i64]([1, 3]) + assert v[1:6:2] == vec[i64]([1, 3, 5]) + assert v[5:1:-2] == vec[i64]([5, 3]) + assert v[6:1:-2] == vec[i64]([6, 4, 2]) + v = vec[i64](range(5)) + assert v[::-1] == vec[i64]([4, 3, 2, 1, 0]) + with assertRaises(ValueError): + v[1:3:0] + +def test_contains() -> None: + v = vec[i64](range(2, 7)) + for i in range(2, 7): + assert i in v + assert 1 not in v + assert 7 not in v + assert -1 not in v + assert 1233453452 not in v + assert -1233453452 not in v + vv = vec[i64]() + assert 3 not in vv + +def test_remove() -> None: + a = [4, 7, 9] + for i in a: + v = vec[i64](a) + v = remove(v, i) + assert v == vec[i64]([j for j in a if j != i]) + v = vec[i64](a) + v = remove(v, 4) + v = remove(v, 7) + v = remove(v, 9) + assert v == vec[i64]() + with assertRaises(ValueError): + remove(v, 4) + v = append(v, 5) + with assertRaises(ValueError): + remove(v, 7) + v = remove(v, 5) + assert len(v) == 0 + v = vec[i64]([1, 1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1]) + v = remove(v, 1) + assert v == vec[i64]() + f: Any = 1.1 + with assertRaises(TypeError): + remove(v, f) + s: Any = 'x' + with assertRaises(TypeError): + remove(v, s) + +def test_pop_last() -> None: + v = vec[i64]([4, 7, 9]) + v, n = pop(v) + assert n == 9 + assert v == vec[i64]([4, 7]) + v, n = pop(v) + assert n == 7 + assert v == vec[i64]([4]) + v, n = pop(v) + assert n == 4 + assert v == vec[i64]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[i64]([4, 7, 9, 15, 22]) + v, n = pop(v, 0) + assert n == 4 + assert v == vec[i64]([7, 9, 15, 22]) + v, n = pop(v, -1) + assert n == 22 + assert v == vec[i64]([7, 9, 15]) + v, n = pop(v, 1) + assert n == 9 + assert v == vec[i64]([7, 15]) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, n = pop(v, -2) + assert n == 7 + assert v == vec[i64]([15]) + v, n = pop(v, 0) + assert n == 15 + assert v == vec[i64]() diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test new file mode 100644 index 0000000000000..459822d6776d0 --- /dev/null +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -0,0 +1,378 @@ +[case testLibrtVecsMiscInterpreted_librt_experimental] +# Test cases for vec[]. using generic operations. +# This simulates use from interpreted code. +# +# All specialized vec types share most of the implementation, so we don't try to +# duplicate all tests in run-vecs-i64-interp.test, which acts also as a more detailed +# test suite for specialized vec types. + +import sys +from typing import Any +import typing + +import librt.vecs +import mypy_extensions + +from testutil import assertRaises + +# Access via Any references to force generic operations +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") +u8: Any = mypy_extensions.u8 +i16: Any = mypy_extensions.i16 +i32: Any = mypy_extensions.i32 +i64: Any = mypy_extensions.i64 + +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + +# TODO: Add some nested vec tests + +ITEM_TYPES = [i64, u8, i16, i32, float, bool] + +# Common tests for all item types + +def test_bare_vec_cannot_be_constructed() -> None: + with assertRaises(TypeError): + vec() + with assertRaises(TypeError): + vec([]) + +def test_vec_indexing() -> None: + for t in ITEM_TYPES: + assert type(vec[t]) is type + t1 = vec[t] + t2 = vec[t] + assert type(vec[t1]) is type(vec[t2]) + _ = [t1, t2] # Ensure t1 and t2 are live here + + assert vec[t] is not vec + if t is not i64: + assert vec[t] is not vec[i64] + else: + assert vec[t] is not vec[i32] + + with assertRaises(TypeError): + vec[Optional[t]] # Not supported + +def test_type_str() -> None: + for t in ITEM_TYPES: + name = t.__name__ + assert str(vec[t]) == f"" + assert repr(vec[t]) == f"" + +def test_construct_empty() -> None: + for t in ITEM_TYPES: + v = vec[t]() + assert type(v) is vec[t] + assert len(v) == 0 + +def test_basic_int_operations() -> None: + # All of the non-bool item types support int values 0 to 255 + for t in ITEM_TYPES: + if t is bool: + continue + v = vec[t]([0, 4, 255]) + assert len(v) == 3 + assert v[0] == 0 + assert v[1] == 4 + assert v[2] == 255 + assert v[-1] == 255 + assert v[-2] == 4 + assert v[-3] == 0 + for bad_index in 3, -4, 1 << 100, -1 << 100: + with assertRaises(IndexError): + v[bad_index] + with assertRaises(TypeError): + v["0"] + with assertRaises(TypeError): + append(v, "0") + +def test_equality() -> None: + for t1 in ITEM_TYPES: + for t2 in ITEM_TYPES: + assert vec[t1]() != [] + if t1 is t2: + assert vec[t1]() == vec[t2]() + else: + assert vec[t1]() != vec[t2]() + +def test_repr() -> None: + for t in ITEM_TYPES: + name = t.__name__ + assert repr(vec[t]()) == f"vec[{name}]([])" + +# vec[float] tests + +def test_float_construct_from_initializer() -> None: + v = vec[float]([123.5]) + assert len(v) == 1 + assert v[0] == 123.5 + v = vec[float](x + 1 for x in [5, 7.5, -8.5, 10]) + assert len(v) == 4 + assert v[0] == 6 + assert v[1] == 8.5 + assert v[2] == -7.5 + assert v[3] == 11 + with assertRaises(TypeError): + vec[float](1) + with assertRaises(TypeError): + vec[float](['1']) + with assertRaises(TypeError): + vec[float]([None]) + +def test_float_append() -> None: + v = vec[float]() + v = append(v, 1.5) + assert str(v) == "vec[float]([1.5])" + v = append(v, -2) + assert str(v) == "vec[float]([1.5, -2.0])" + +def test_float_get_item() -> None: + v = append(vec[float](), 44.5) + assert v[0] == 44.5 + v = append(v, -56.5) + assert v[0] == 44.5 + assert v[1] == -56.5 + +def test_float_remove() -> None: + a = [4.5, 7, 9] + for x in a: + v = vec[float](a) + v = remove(v, x) + assert v == vec[float]([y for y in a if y != x]) + +def test_float_pop() -> None: + v = vec[float]([4.5, 7, 9.5]) + v, n = pop(v) + assert n == 9.5 + assert v == vec[float]([4.5, 7]) + +# vec[u8] tests + +def test_u8_construct_from_initializer() -> None: + v = vec[u8]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[u8](x for x in [0, 5, 255, 10]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 255 + assert v[3] == 10 + with assertRaises(TypeError): + vec[u8](1) + with assertRaises(TypeError): + vec[u8](['1']) + with assertRaises(TypeError): + vec[u8]([None]) + +def test_u8_append() -> None: + v = vec[u8]() + v = append(v, 5) + assert str(v) == "vec[u8]([5])" + v = append(v, 255) + assert str(v) == "vec[u8]([5, 255])" + +def test_u8_item_overflow() -> None: + with assertRaises(OverflowError): + vec[u8]([256]) + with assertRaises(OverflowError): + vec[u8]([-1]) + with assertRaises(OverflowError): + vec[u8]([2**100]) + with assertRaises(OverflowError): + vec[u8]([-2**100]) + +def test_u8_get_item() -> None: + v = append(vec[u8](), 44) + assert v[0] == 44 + v = append(v, 238) + assert v[0] == 44 + assert v[1] == 238 + +def test_u8_remove() -> None: + a = [0, 7, 255] + for x in a: + v = vec[u8](a) + v = remove(v, x) + assert v == vec[u8]([y for y in a if y != x]) + +def test_u8_pop() -> None: + v = vec[u8]([0, 7, 255]) + v, n = pop(v) + assert n == 255 + assert v == vec[u8]([0, 7]) + +# vec[i16] tests + +def test_i16_construct_from_initializer() -> None: + v = vec[i16]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i16](x for x in [0, 5, 32767, -32768]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 32767 + assert v[3] == -32768 + with assertRaises(TypeError): + vec[i16](1) + with assertRaises(TypeError): + vec[i16](['1']) + with assertRaises(TypeError): + vec[i16]([None]) + +def test_i16_append() -> None: + v = vec[i16]() + v = append(v, 5) + assert str(v) == "vec[i16]([5])" + v = append(v, 32767) + assert str(v) == "vec[i16]([5, 32767])" + +def test_i16_item_overflow() -> None: + with assertRaises(OverflowError): + vec[i16]([32768]) + with assertRaises(OverflowError): + vec[i16]([-32769]) + with assertRaises(OverflowError): + vec[i16]([2**100]) + with assertRaises(OverflowError): + vec[i16]([-2**100]) + +def test_i16_get_item() -> None: + v = append(vec[i16](), 44) + assert v[0] == 44 + v = append(v, -238) + assert v[0] == 44 + assert v[1] == -238 + +def test_i16_remove() -> None: + a = [0, 7, -532, 32767] + for x in a: + v = vec[i16](a) + v = remove(v, x) + assert v == vec[i16]([y for y in a if y != x]) + +def test_i16_pop() -> None: + v = vec[i16]([0, -7, 32767]) + v, n = pop(v) + assert n == 32767 + assert v == vec[i16]([0, -7]) + +# vec[i32] tests + +def test_i32_construct_from_initializer() -> None: + v = vec[i32]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i32](x for x in [0, 5, 2**31 - 1, -2**31]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 2**31 - 1 + assert v[3] == -2**31 + with assertRaises(TypeError): + vec[i32](1) + with assertRaises(TypeError): + vec[i32](['1']) + with assertRaises(TypeError): + vec[i32]([None]) + +def test_i32_append() -> None: + v = vec[i32]() + v = append(v, 5) + assert str(v) == "vec[i32]([5])" + v = append(v, 2**31 - 1) + assert str(v) == "vec[i32]([5, 2147483647])" + +def test_i32_item_overflow() -> None: + with assertRaises(OverflowError): + vec[i32]([2**31]) + with assertRaises(OverflowError): + vec[i32]([-2**31 - 1]) + with assertRaises(OverflowError): + vec[i32]([2**100]) + with assertRaises(OverflowError): + vec[i32]([-2**100]) + +def test_i32_get_item() -> None: + v = append(vec[i32](), 44) + assert v[0] == 44 + v = append(v, -238) + assert v[0] == 44 + assert v[1] == -238 + +def test_i32_remove() -> None: + a = [0, 7, -532, 2**31 - 1] + for x in a: + v = vec[i32](a) + v = remove(v, x) + assert v == vec[i32]([y for y in a if y != x]) + +def test_i32_pop() -> None: + v = vec[i32]([0, -7, 2**31 - 1]) + v, n = pop(v) + assert n == 2**31 - 1 + assert v == vec[i32]([0, -7]) + +# vec[bool] tests + +def test_bool_construct_from_initializer() -> None: + v = vec[bool]([True]) + assert len(v) == 1 + assert v[0] == True + v = vec[bool](x for x in [True, False, True]) + assert len(v) == 3 + assert v[0] is True + assert v[1] is False + assert v[2] is True + with assertRaises(TypeError): + vec[bool](1) + with assertRaises(TypeError): + vec[bool](['1']) + with assertRaises(TypeError): + vec[bool]([1]) + with assertRaises(TypeError): + vec[bool]([None]) + +def test_bool_append() -> None: + v = vec[bool]() + v = append(v, False) + assert str(v) == "vec[bool]([False])" + v = append(v, True) + assert str(v) == "vec[bool]([False, True])" + +def test_bool_get_item() -> None: + v = append(vec[bool](), True) + assert v[0] is True + v = append(v, False) + assert v[0] is True + assert v[1] is False + +def test_bool_remove() -> None: + a = [True, False] + for x in a: + v = vec[bool](a) + v = remove(v, x) + assert v == vec[bool]([y for y in a if y != x]) + +def test_bool_pop() -> None: + v = vec[bool]([True, False, True]) + v, n = pop(v) + assert n is True + assert v == vec[bool]([True, False]) + +[case testLibrtVecsFeaturesNotAvailableInNonExperimentalBuild_librt] +# This also ensures librt.vecs can be built without experimental features +import librt.vecs + +def test_features_not_available() -> None: + vecs: object = getattr(librt, "vecs") + assert not hasattr(vecs, "vec") + assert not hasattr(vecs, "append") + assert not hasattr(vecs, "remove") + assert not hasattr(vecs, "pop") diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 8157d741bf084..9dbd3db1e8a8e 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -78,6 +78,8 @@ "run-librt-strings.test", "run-base64.test", "run-match.test", + "run-vecs-i64-interp.test", + "run-vecs-misc-interp.test", ] if sys.version_info >= (3, 12):