From b685c2ce552148824ae6c99546794f4f4f6f4096 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 12:58:41 +0100 Subject: [PATCH 001/208] Add initial C extension --- mypyc/lib-rt/vecs/vecs.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 mypyc/lib-rt/vecs/vecs.c diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c new file mode 100644 index 0000000000000..bd21c6ddc5482 --- /dev/null +++ b/mypyc/lib-rt/vecs/vecs.c @@ -0,0 +1,35 @@ +#define PY_SSIZE_T_CLEAN +#include + +static PyObject * +vecs_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +static PyMethodDef VecsMethods[] = { + {"system", vecs_system, METH_VARARGS, + "Execute a shell command."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static struct PyModuleDef vecsmodule = { + PyModuleDef_HEAD_INIT, + "vecs", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + VecsMethods +}; + +PyMODINIT_FUNC +PyInit_vecs(void) +{ + return PyModule_Create(&vecsmodule); +} From 11188d873b41a1cc068ea5f618799e9a45673c0a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 13:03:09 +0100 Subject: [PATCH 002/208] Add empty vec class --- mypyc/lib-rt/vecs/vecs.c | 54 ++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index bd21c6ddc5482..61f089f122680 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -1,35 +1,45 @@ #define PY_SSIZE_T_CLEAN #include -static PyObject * -vecs_system(PyObject *self, PyObject *args) -{ - const char *command; - int sts; +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ +} VecObject; - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); -} - -static PyMethodDef VecsMethods[] = { - {"system", vecs_system, METH_VARARGS, - "Execute a shell command."}, - {NULL, NULL, 0, NULL} /* Sentinel */ +static PyTypeObject VecType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecs.vec", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, }; -static struct PyModuleDef vecsmodule = { +static PyModuleDef vecsmodule = { PyModuleDef_HEAD_INIT, - "vecs", /* name of module */ - NULL, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - VecsMethods + .m_name = "vecs", + .m_doc = "vecs doc", + .m_size = -1, }; PyMODINIT_FUNC PyInit_vecs(void) { - return PyModule_Create(&vecsmodule); + PyObject *m; + if (PyType_Ready(&VecType) < 0) + return NULL; + + m = PyModule_Create(&vecsmodule); + if (m == NULL) + return NULL; + + Py_INCREF(&VecType); + if (PyModule_AddObject(m, "vec", (PyObject *) &VecType) < 0) { + Py_DECREF(&VecType); + Py_DECREF(m); + return NULL; + } + + return m; } From 37857c97f02aa592501dafbc45d9180d3ac5a7da Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 13:11:01 +0100 Subject: [PATCH 003/208] Add dummy append function --- mypyc/lib-rt/vecs/vecs.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 61f089f122680..c7ba2d440e5d7 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -16,11 +16,29 @@ static PyTypeObject VecType = { .tp_new = PyType_GenericNew, }; +static PyObject * +vecs_append(PyObject *self, PyObject *args) +{ + PyObject *obj; + long long x; + + if (!PyArg_ParseTuple(args, "OL", &obj, &x)) + return NULL; + Py_INCREF(obj); + return obj; +} + +static PyMethodDef VecsMethods[] = { + {"append", vecs_append, METH_VARARGS, "Append a value to a vec"}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + static PyModuleDef vecsmodule = { PyModuleDef_HEAD_INIT, .m_name = "vecs", .m_doc = "vecs doc", .m_size = -1, + .m_methods = VecsMethods, }; PyMODINIT_FUNC From c49c01922bce5c9585df72dfbb85505d8dbfa56b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 15:14:30 +0100 Subject: [PATCH 004/208] Can construct empty vec --- mypyc/lib-rt/vecs/vecs.c | 48 +++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index c7ba2d440e5d7..d15ccbebc6ea4 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -1,21 +1,59 @@ #define PY_SSIZE_T_CLEAN #include +static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw); + typedef struct { - PyObject_HEAD - /* Type-specific fields go here. */ + PyObject_VAR_HEAD + Py_ssize_t capacity; + long long item[1]; } VecObject; static PyTypeObject VecType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecs.vec", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecObject), - .tp_itemsize = 0, + .tp_basicsize = sizeof(VecObject) - sizeof(long long), + .tp_itemsize = sizeof(long long), .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, + .tp_new = vec_new, + .tp_free = PyObject_Del, }; +static VecObject * +vec_alloc(Py_ssize_t size) +{ + VecObject *v; + /* TODO: Check for overflow */ + v = PyObject_NewVar(VecObject, &VecType, size); + if (v == NULL) + return NULL; + return v; +} + +PyObject * +Vec_New(Py_ssize_t size) +{ + VecObject *v; + v = vec_alloc(size); + if (v == NULL) + return NULL; + for (Py_ssize_t i = 0; i < size; i++) { + v->item[i] = 0; + } + v->capacity = size; + return (PyObject *)v; +} + +PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { + static char *kwlist[] = {"", NULL}; + PyObject *t; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O:vec", kwlist, &t)) { + return NULL; + } + return Vec_New(0); +} + static PyObject * vecs_append(PyObject *self, PyObject *args) { From 0ee2a949826eda6d51fc4a0c336e90816e83176c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 17:56:49 +0100 Subject: [PATCH 005/208] Implement vec operations (indexing, append, len) --- mypyc/lib-rt/vecs/vecs.c | 85 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index d15ccbebc6ea4..7593710adf8fb 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -1,14 +1,61 @@ #define PY_SSIZE_T_CLEAN #include +#define VEC_SIZE(v) ((v)->ob_base.ob_size) + static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw); typedef struct { PyObject_VAR_HEAD - Py_ssize_t capacity; - long long item[1]; + Py_ssize_t len; + long long items[1]; } VecObject; +PyObject *vec_repr(PyObject *self) { + // TODO: Type check, refcounting, error handling + VecObject *o = (VecObject *)self; + PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); + PyObject *suffix = Py_BuildValue("s", "])"); + PyObject *l = Py_BuildValue("[]"); + PyObject *sep = Py_BuildValue("s", ""); + PyObject *comma = Py_BuildValue("s", ", "); + PyList_Append(l, prefix); + for (int i = 0; i < o->len; i++) { + char s[100]; + sprintf(s, "%lld", o->items[i]); + PyObject *x = Py_BuildValue("s", s); + PyList_Append(l, x); + if (i + 1 < o->len) + PyList_Append(l, comma); + } + PyList_Append(l, suffix); + return PyUnicode_Join(sep, l); +} + +PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { + // TODO: Type check o + VecObject *v = (VecObject *)o; + if ((size_t)i < (size_t)v->len) { + return PyLong_FromLongLong(v->items[i]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +Py_ssize_t vec_length(PyObject *o) { + // TODO: Type check o + return ((VecObject *)o)->len; +} + +static PyMappingMethods VecMapping = { + .mp_length = vec_length, +}; + +static PySequenceMethods VecSequence = { + .sq_item = vec_get_item, +}; + static PyTypeObject VecType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecs.vec", @@ -18,6 +65,9 @@ static PyTypeObject VecType = { .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = vec_new, .tp_free = PyObject_Del, + .tp_repr = (reprfunc)vec_repr, + .tp_as_sequence = &VecSequence, + .tp_as_mapping = &VecMapping, }; static VecObject * @@ -39,9 +89,9 @@ Vec_New(Py_ssize_t size) if (v == NULL) return NULL; for (Py_ssize_t i = 0; i < size; i++) { - v->item[i] = 0; + v->items[i] = 0; } - v->capacity = size; + v->len = size; return (PyObject *)v; } @@ -54,6 +104,27 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return Vec_New(0); } +VecObject * +Vec_Append(VecObject *vec, long long x) { + Py_ssize_t cap = VEC_SIZE(vec); + Py_ssize_t len = vec->len; + if (len < cap) { + vec->items[len] = x; + vec->len = len + 1; + Py_INCREF(vec); + return vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + VecObject *new = vec_alloc(new_size); + if (new == NULL) + return NULL; + memcpy(new->items, vec->items, sizeof(long long) * len); + new->items[len] = x; + new->len = len + 1; + return new; + } +} + static PyObject * vecs_append(PyObject *self, PyObject *args) { @@ -62,8 +133,10 @@ vecs_append(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OL", &obj, &x)) return NULL; - Py_INCREF(obj); - return obj; + + // TODO: Type check obj + + return (PyObject *)Vec_Append((VecObject *)obj, x); } static PyMethodDef VecsMethods[] = { From bc61aa6f95db29e417fe919dfe15c319620b3725 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 5 Apr 2021 18:17:21 +0100 Subject: [PATCH 006/208] Support setting vec item --- mypyc/lib-rt/vecs/vecs.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 7593710adf8fb..39a9e46efb26b 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -43,6 +43,21 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } +int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + // TODO: Type check o + VecObject *v = (VecObject *)self; + if ((size_t)i < (size_t)v->len) { + long long x = PyLong_AsLongLong(o); + if (x == -1 && PyErr_Occurred()) + return -1; + v->items[i] = x; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o return ((VecObject *)o)->len; @@ -54,6 +69,7 @@ static PyMappingMethods VecMapping = { static PySequenceMethods VecSequence = { .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, }; static PyTypeObject VecType = { From dbbb66ee4769ad0165d851d361f7aa1f453a91b1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Apr 2021 15:55:00 +0100 Subject: [PATCH 007/208] Create basic capsule --- mypyc/lib-rt/vecs/vecs.c | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 39a9e46efb26b..a5e52b6263fda 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -11,6 +11,24 @@ typedef struct { long long items[1]; } VecObject; +// vec[i64] operations + type object (stored in a capsule) +typedef struct { + PyTypeObject *type; + PyObject *(*alloc)(Py_ssize_t); + VecObject *(*append)(VecObject *, int64_t); + // VecObject *(*extend)(VecObject *, PyObject *); + // VecObject *(*slice)(VecObject *, int64_t, int64_t); + // VecObject *(*concat)(VecObject *, VecObject *); + // bool (*contains)(VecObject *, int64_t); + // int64_t (*pop)(VecObject *); + // iter? +} VecI64Features; + +typedef struct { + VecI64Features *i64; +} VecCapsule; + + PyObject *vec_repr(PyObject *self) { // TODO: Type check, refcounting, error handling VecObject *o = (VecObject *)self; @@ -121,7 +139,7 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } VecObject * -Vec_Append(VecObject *vec, long long x) { +Vec_Append(VecObject *vec, int64_t x) { Py_ssize_t cap = VEC_SIZE(vec); Py_ssize_t len = vec->len; if (len < cap) { @@ -168,6 +186,16 @@ static PyModuleDef vecsmodule = { .m_methods = VecsMethods, }; +static VecI64Features I64Features = { + &VecType, + Vec_New, + Vec_Append, +}; + +static VecCapsule Capsule = { + &I64Features +}; + PyMODINIT_FUNC PyInit_vecs(void) { @@ -186,5 +214,8 @@ PyInit_vecs(void) return NULL; } + if (PyCapsule_New(&Capsule, "vecs.capsule", NULL) == NULL) + return NULL; + return m; } From 7d99cb5bd643fcfd1be7e37fa04abf1126a541b6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Apr 2021 16:11:26 +0100 Subject: [PATCH 008/208] Move some declarations to a header --- mypyc/lib-rt/vecs/vecs.c | 25 +------------------------ mypyc/lib-rt/vecs/vecs.h | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 24 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vecs.h diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index a5e52b6263fda..370182f1e5c61 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -1,34 +1,11 @@ #define PY_SSIZE_T_CLEAN #include +#include "vecs.h" #define VEC_SIZE(v) ((v)->ob_base.ob_size) static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw); -typedef struct { - PyObject_VAR_HEAD - Py_ssize_t len; - long long items[1]; -} VecObject; - -// vec[i64] operations + type object (stored in a capsule) -typedef struct { - PyTypeObject *type; - PyObject *(*alloc)(Py_ssize_t); - VecObject *(*append)(VecObject *, int64_t); - // VecObject *(*extend)(VecObject *, PyObject *); - // VecObject *(*slice)(VecObject *, int64_t, int64_t); - // VecObject *(*concat)(VecObject *, VecObject *); - // bool (*contains)(VecObject *, int64_t); - // int64_t (*pop)(VecObject *); - // iter? -} VecI64Features; - -typedef struct { - VecI64Features *i64; -} VecCapsule; - - PyObject *vec_repr(PyObject *self) { // TODO: Type check, refcounting, error handling VecObject *o = (VecObject *)self; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h new file mode 100644 index 0000000000000..85a006256ff22 --- /dev/null +++ b/mypyc/lib-rt/vecs/vecs.h @@ -0,0 +1,30 @@ +#ifndef VEC_H_INCL +#define VEC_H_INCL + +#define PY_SSIZE_T_CLEAN +#include + +typedef struct { + PyObject_VAR_HEAD + Py_ssize_t len; + long long items[1]; +} VecObject; + +// vec[i64] operations + type object (stored in a capsule) +typedef struct { + PyTypeObject *type; + PyObject *(*alloc)(Py_ssize_t); + VecObject *(*append)(VecObject *, int64_t); + // VecObject *(*extend)(VecObject *, PyObject *); + // VecObject *(*slice)(VecObject *, int64_t, int64_t); + // VecObject *(*concat)(VecObject *, VecObject *); + // bool (*contains)(VecObject *, int64_t); + // int64_t (*pop)(VecObject *); + // iter? +} VecI64Features; + +typedef struct { + VecI64Features *i64; +} VecCapsule; + +#endif From 68283398f875e70b2e852397a88f6bd3e62f8036 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Apr 2021 17:04:30 +0100 Subject: [PATCH 009/208] Tweak types --- mypyc/lib-rt/vecs/vecs.c | 11 ++++++----- mypyc/lib-rt/vecs/vecs.h | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 370182f1e5c61..6466f2dd0d1ba 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -115,15 +115,16 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return Vec_New(0); } -VecObject * -Vec_Append(VecObject *vec, int64_t x) { +PyObject * +Vec_Append(PyObject *obj, int64_t x) { + VecObject *vec = (VecObject *)obj; Py_ssize_t cap = VEC_SIZE(vec); Py_ssize_t len = vec->len; if (len < cap) { vec->items[len] = x; vec->len = len + 1; Py_INCREF(vec); - return vec; + return (PyObject *)vec; } else { Py_ssize_t new_size = 2 * cap + 1; VecObject *new = vec_alloc(new_size); @@ -132,7 +133,7 @@ Vec_Append(VecObject *vec, int64_t x) { memcpy(new->items, vec->items, sizeof(long long) * len); new->items[len] = x; new->len = len + 1; - return new; + return (PyObject *)new; } } @@ -147,7 +148,7 @@ vecs_append(PyObject *self, PyObject *args) // TODO: Type check obj - return (PyObject *)Vec_Append((VecObject *)obj, x); + return Vec_Append(obj, x); } static PyMethodDef VecsMethods[] = { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 85a006256ff22..0e58975912443 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -4,17 +4,17 @@ #define PY_SSIZE_T_CLEAN #include -typedef struct { +typedef struct _VecObject { PyObject_VAR_HEAD Py_ssize_t len; long long items[1]; } VecObject; // vec[i64] operations + type object (stored in a capsule) -typedef struct { +typedef struct _VecI64Features { PyTypeObject *type; PyObject *(*alloc)(Py_ssize_t); - VecObject *(*append)(VecObject *, int64_t); + PyObject *(*append)(PyObject *, int64_t); // VecObject *(*extend)(VecObject *, PyObject *); // VecObject *(*slice)(VecObject *, int64_t, int64_t); // VecObject *(*concat)(VecObject *, VecObject *); From a771ac0be83a65486349d995b3ce2bc7eb0b4d0b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Apr 2021 17:30:59 +0100 Subject: [PATCH 010/208] Fix capsule storage --- mypyc/lib-rt/vecs/vecs.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 6466f2dd0d1ba..28e9b4abc595f 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -192,8 +192,16 @@ PyInit_vecs(void) return NULL; } - if (PyCapsule_New(&Capsule, "vecs.capsule", NULL) == NULL) + PyObject *c_api = PyCapsule_New(&Capsule, "vecs._C_API", NULL); + if (c_api == NULL) return NULL; + if (PyModule_AddObject(m, "_C_API", c_api) < 0) { + Py_XDECREF(c_api); + Py_DECREF(&VecType); + Py_DECREF(m); + return NULL; + } + return m; } From 152e66b79d1afef81bf49cf66b1495ce59ac82ee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 10:50:15 +0000 Subject: [PATCH 011/208] Minor refactoring --- mypyc/lib-rt/vecs/vecs.c | 25 +++++++++++++------------ mypyc/lib-rt/vecs/vecs.h | 17 +++++++++-------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 28e9b4abc595f..cd7fb75854934 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -8,7 +8,7 @@ static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw); PyObject *vec_repr(PyObject *self) { // TODO: Type check, refcounting, error handling - VecObject *o = (VecObject *)self; + VecI64Object *o = (VecI64Object *)self; PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); PyObject *suffix = Py_BuildValue("s", "])"); PyObject *l = Py_BuildValue("[]"); @@ -29,7 +29,7 @@ PyObject *vec_repr(PyObject *self) { PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { // TODO: Type check o - VecObject *v = (VecObject *)o; + VecI64Object *v = (VecI64Object *)o; if ((size_t)i < (size_t)v->len) { return PyLong_FromLongLong(v->items[i]); } else { @@ -40,7 +40,7 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { // TODO: Type check o - VecObject *v = (VecObject *)self; + VecI64Object *v = (VecI64Object *)self; if ((size_t)i < (size_t)v->len) { long long x = PyLong_AsLongLong(o); if (x == -1 && PyErr_Occurred()) @@ -55,7 +55,7 @@ int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o - return ((VecObject *)o)->len; + return ((VecI64Object *)o)->len; } static PyMappingMethods VecMapping = { @@ -71,7 +71,7 @@ static PyTypeObject VecType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecs.vec", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecObject) - sizeof(long long), + .tp_basicsize = sizeof(VecI64Object) - sizeof(long long), .tp_itemsize = sizeof(long long), .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = vec_new, @@ -81,12 +81,12 @@ static PyTypeObject VecType = { .tp_as_mapping = &VecMapping, }; -static VecObject * +static VecI64Object * vec_alloc(Py_ssize_t size) { - VecObject *v; + VecI64Object *v; /* TODO: Check for overflow */ - v = PyObject_NewVar(VecObject, &VecType, size); + v = PyObject_NewVar(VecI64Object, &VecType, size); if (v == NULL) return NULL; return v; @@ -95,7 +95,7 @@ vec_alloc(Py_ssize_t size) PyObject * Vec_New(Py_ssize_t size) { - VecObject *v; + VecI64Object *v; v = vec_alloc(size); if (v == NULL) return NULL; @@ -117,22 +117,22 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { PyObject * Vec_Append(PyObject *obj, int64_t x) { - VecObject *vec = (VecObject *)obj; + VecI64Object *vec = (VecI64Object *)obj; Py_ssize_t cap = VEC_SIZE(vec); Py_ssize_t len = vec->len; if (len < cap) { vec->items[len] = x; vec->len = len + 1; - Py_INCREF(vec); return (PyObject *)vec; } else { Py_ssize_t new_size = 2 * cap + 1; - VecObject *new = vec_alloc(new_size); + VecI64Object *new = vec_alloc(new_size); if (new == NULL) return NULL; memcpy(new->items, vec->items, sizeof(long long) * len); new->items[len] = x; new->len = len + 1; + Py_DECREF(vec); return (PyObject *)new; } } @@ -148,6 +148,7 @@ vecs_append(PyObject *self, PyObject *args) // TODO: Type check obj + Py_INCREF(obj); return Vec_Append(obj, x); } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 0e58975912443..6499ebc368b78 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -4,22 +4,23 @@ #define PY_SSIZE_T_CLEAN #include -typedef struct _VecObject { +typedef struct _VecI64Object { PyObject_VAR_HEAD Py_ssize_t len; - long long items[1]; -} VecObject; + int64_t items[1]; +} VecI64Object; // vec[i64] operations + type object (stored in a capsule) typedef struct _VecI64Features { PyTypeObject *type; PyObject *(*alloc)(Py_ssize_t); PyObject *(*append)(PyObject *, int64_t); - // VecObject *(*extend)(VecObject *, PyObject *); - // VecObject *(*slice)(VecObject *, int64_t, int64_t); - // VecObject *(*concat)(VecObject *, VecObject *); - // bool (*contains)(VecObject *, int64_t); - // int64_t (*pop)(VecObject *); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*slice)(PyObject *, int64_t, int64_t); + // PyObject *(*concat)(PyObject *, VecObject *); + // bool (*contains)(PyObject *, int64_t); + // int64_t (*pop)(PyObject *); + // bool (*remove)(PyObject *, int64_t); // iter? } VecI64Features; From 8bf98759aa3d4bd7a3dbbd86540530e3357f00c7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 11:12:13 +0000 Subject: [PATCH 012/208] More refactoring --- mypyc/lib-rt/vecs/vecs.c | 64 ++++++++++++++++++++-------------------- mypyc/lib-rt/vecs/vecs.h | 26 +++++++++++++++- 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index cd7fb75854934..27a2023455f33 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -4,9 +4,9 @@ #define VEC_SIZE(v) ((v)->ob_base.ob_size) -static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw); +static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); -PyObject *vec_repr(PyObject *self) { +PyObject *vec_i64_repr(PyObject *self) { // TODO: Type check, refcounting, error handling VecI64Object *o = (VecI64Object *)self; PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); @@ -17,7 +17,7 @@ PyObject *vec_repr(PyObject *self) { PyList_Append(l, prefix); for (int i = 0; i < o->len; i++) { char s[100]; - sprintf(s, "%lld", o->items[i]); + sprintf(s, "%lld", (long long)o->items[i]); PyObject *x = Py_BuildValue("s", s); PyList_Append(l, x); if (i + 1 < o->len) @@ -27,7 +27,7 @@ PyObject *vec_repr(PyObject *self) { return PyUnicode_Join(sep, l); } -PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { +PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { // TODO: Type check o VecI64Object *v = (VecI64Object *)o; if ((size_t)i < (size_t)v->len) { @@ -38,7 +38,7 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { +int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { // TODO: Type check o VecI64Object *v = (VecI64Object *)self; if ((size_t)i < (size_t)v->len) { @@ -58,45 +58,45 @@ Py_ssize_t vec_length(PyObject *o) { return ((VecI64Object *)o)->len; } -static PyMappingMethods VecMapping = { +static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, }; -static PySequenceMethods VecSequence = { - .sq_item = vec_get_item, - .sq_ass_item = vec_ass_item, +static PySequenceMethods VecI64Sequence = { + .sq_item = vec_i64_get_item, + .sq_ass_item = vec_i64_ass_item, }; -static PyTypeObject VecType = { +static PyTypeObject VecI64Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecs.vec", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object) - sizeof(long long), - .tp_itemsize = sizeof(long long), + .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), + .tp_itemsize = sizeof(int64_t), .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = vec_new, + .tp_new = vec_i64_new, .tp_free = PyObject_Del, - .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecSequence, - .tp_as_mapping = &VecMapping, + .tp_repr = (reprfunc)vec_i64_repr, + .tp_as_sequence = &VecI64Sequence, + .tp_as_mapping = &VecI64Mapping, }; static VecI64Object * -vec_alloc(Py_ssize_t size) +vec_i64_alloc(Py_ssize_t size) { VecI64Object *v; /* TODO: Check for overflow */ - v = PyObject_NewVar(VecI64Object, &VecType, size); + v = PyObject_NewVar(VecI64Object, &VecI64Type, size); if (v == NULL) return NULL; return v; } PyObject * -Vec_New(Py_ssize_t size) +Vec_I64_New(Py_ssize_t size) { VecI64Object *v; - v = vec_alloc(size); + v = vec_i64_alloc(size); if (v == NULL) return NULL; for (Py_ssize_t i = 0; i < size; i++) { @@ -106,13 +106,13 @@ Vec_New(Py_ssize_t size) return (PyObject *)v; } -PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { +PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { static char *kwlist[] = {"", NULL}; PyObject *t; if (!PyArg_ParseTupleAndKeywords(args, kw, "O:vec", kwlist, &t)) { return NULL; } - return Vec_New(0); + return Vec_I64_New(0); } PyObject * @@ -126,10 +126,10 @@ Vec_Append(PyObject *obj, int64_t x) { return (PyObject *)vec; } else { Py_ssize_t new_size = 2 * cap + 1; - VecI64Object *new = vec_alloc(new_size); + VecI64Object *new = vec_i64_alloc(new_size); if (new == NULL) return NULL; - memcpy(new->items, vec->items, sizeof(long long) * len); + memcpy(new->items, vec->items, sizeof(int64_t) * len); new->items[len] = x; new->len = len + 1; Py_DECREF(vec); @@ -141,7 +141,7 @@ static PyObject * vecs_append(PyObject *self, PyObject *args) { PyObject *obj; - long long x; + int64_t x; if (!PyArg_ParseTuple(args, "OL", &obj, &x)) return NULL; @@ -166,8 +166,8 @@ static PyModuleDef vecsmodule = { }; static VecI64Features I64Features = { - &VecType, - Vec_New, + &VecI64Type, + Vec_I64_New, Vec_Append, }; @@ -179,16 +179,16 @@ PyMODINIT_FUNC PyInit_vecs(void) { PyObject *m; - if (PyType_Ready(&VecType) < 0) + if (PyType_Ready(&VecI64Type) < 0) return NULL; m = PyModule_Create(&vecsmodule); if (m == NULL) return NULL; - Py_INCREF(&VecType); - if (PyModule_AddObject(m, "vec", (PyObject *) &VecType) < 0) { - Py_DECREF(&VecType); + Py_INCREF(&VecI64Type); + if (PyModule_AddObject(m, "vec", (PyObject *) &VecI64Type) < 0) { + Py_DECREF(&VecI64Type); Py_DECREF(m); return NULL; } @@ -199,7 +199,7 @@ PyInit_vecs(void) if (PyModule_AddObject(m, "_C_API", c_api) < 0) { Py_XDECREF(c_api); - Py_DECREF(&VecType); + Py_DECREF(&VecI64Type); Py_DECREF(m); return NULL; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 6499ebc368b78..d9ccfcf4f7b40 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -17,15 +17,39 @@ typedef struct _VecI64Features { PyObject *(*append)(PyObject *, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); - // PyObject *(*concat)(PyObject *, VecObject *); + // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); // int64_t (*pop)(PyObject *); // bool (*remove)(PyObject *, int64_t); // iter? } VecI64Features; +typedef struct _VecTObject { + PyObject_VAR_HEAD + Py_ssize_t len; + PyObject *type_tagged; + PyObject *items[1]; +} VecTObject; + +// vec[T] operations + type object (stored in a capsule) +// +// T can be a class type, T | None or vec[T] (or a combination of these) +typedef struct _VecTFeatures { + PyTypeObject *type; + PyObject *(*alloc)(Py_ssize_t); + PyObject *(*append)(PyObject *, PyObject *); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*slice)(PyObject *, int64_t, int64_t); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, PyObject *); + // PyObject *(*pop)(PyObject *); + // bool (*remove)(PyObject *, PyObject *); + // iter? +} VecTFeatures; + typedef struct { VecI64Features *i64; + // VecTFeatures *i64; } VecCapsule; #endif From 5503a7a81f97fa374e87cfe0014b886740d41d9c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 11:21:35 +0000 Subject: [PATCH 013/208] Move vec[i64] related stuff to a new file --- mypyc/lib-rt/vecs/vec_i64.c | 139 ++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 146 +----------------------------------- mypyc/lib-rt/vecs/vecs.h | 7 ++ 3 files changed, 148 insertions(+), 144 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_i64.c diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c new file mode 100644 index 0000000000000..e7ea60bc64b7b --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -0,0 +1,139 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "vecs.h" + +static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); + +PyObject *vec_i64_repr(PyObject *self) { + // TODO: Type check, refcounting, error handling + VecI64Object *o = (VecI64Object *)self; + PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); + PyObject *suffix = Py_BuildValue("s", "])"); + PyObject *l = Py_BuildValue("[]"); + PyObject *sep = Py_BuildValue("s", ""); + PyObject *comma = Py_BuildValue("s", ", "); + PyList_Append(l, prefix); + for (int i = 0; i < o->len; i++) { + char s[100]; + sprintf(s, "%lld", (long long)o->items[i]); + PyObject *x = Py_BuildValue("s", s); + PyList_Append(l, x); + if (i + 1 < o->len) + PyList_Append(l, comma); + } + PyList_Append(l, suffix); + return PyUnicode_Join(sep, l); +} + +PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { + // TODO: Type check o + VecI64Object *v = (VecI64Object *)o; + if ((size_t)i < (size_t)v->len) { + return PyLong_FromLongLong(v->items[i]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + // TODO: Type check o + VecI64Object *v = (VecI64Object *)self; + if ((size_t)i < (size_t)v->len) { + long long x = PyLong_AsLongLong(o); + if (x == -1 && PyErr_Occurred()) + return -1; + v->items[i] = x; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + +Py_ssize_t vec_length(PyObject *o) { + // TODO: Type check o + return ((VecI64Object *)o)->len; +} + +static PyMappingMethods VecI64Mapping = { + .mp_length = vec_length, +}; + +static PySequenceMethods VecI64Sequence = { + .sq_item = vec_i64_get_item, + .sq_ass_item = vec_i64_ass_item, +}; + +PyTypeObject VecI64Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecs.vec", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), + .tp_itemsize = sizeof(int64_t), + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vec_i64_new, + .tp_free = PyObject_Del, + .tp_repr = (reprfunc)vec_i64_repr, + .tp_as_sequence = &VecI64Sequence, + .tp_as_mapping = &VecI64Mapping, +}; + +static VecI64Object *vec_i64_alloc(Py_ssize_t size) +{ + VecI64Object *v; + /* TODO: Check for overflow */ + v = PyObject_NewVar(VecI64Object, &VecI64Type, size); + if (v == NULL) + return NULL; + return v; +} + +PyObject *Vec_I64_New(Py_ssize_t size) +{ + VecI64Object *v; + v = vec_i64_alloc(size); + if (v == NULL) + return NULL; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = 0; + } + v->len = size; + return (PyObject *)v; +} + +PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { + static char *kwlist[] = {"", NULL}; + PyObject *t; + if (!PyArg_ParseTupleAndKeywords(args, kw, "O:vec", kwlist, &t)) { + return NULL; + } + return Vec_I64_New(0); +} + +PyObject *Vec_I64_Append(PyObject *obj, int64_t x) { + VecI64Object *vec = (VecI64Object *)obj; + Py_ssize_t cap = VEC_SIZE(vec); + Py_ssize_t len = vec->len; + if (len < cap) { + vec->items[len] = x; + vec->len = len + 1; + return (PyObject *)vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + VecI64Object *new = vec_i64_alloc(new_size); + if (new == NULL) + return NULL; + memcpy(new->items, vec->items, sizeof(int64_t) * len); + new->items[len] = x; + new->len = len + 1; + Py_DECREF(vec); + return (PyObject *)new; + } +} + +VecI64Features I64Features = { + &VecI64Type, + Vec_I64_New, + Vec_I64_Append, +}; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 27a2023455f33..9df17126fde50 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -2,143 +2,7 @@ #include #include "vecs.h" -#define VEC_SIZE(v) ((v)->ob_base.ob_size) - -static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); - -PyObject *vec_i64_repr(PyObject *self) { - // TODO: Type check, refcounting, error handling - VecI64Object *o = (VecI64Object *)self; - PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); - PyObject *suffix = Py_BuildValue("s", "])"); - PyObject *l = Py_BuildValue("[]"); - PyObject *sep = Py_BuildValue("s", ""); - PyObject *comma = Py_BuildValue("s", ", "); - PyList_Append(l, prefix); - for (int i = 0; i < o->len; i++) { - char s[100]; - sprintf(s, "%lld", (long long)o->items[i]); - PyObject *x = Py_BuildValue("s", s); - PyList_Append(l, x); - if (i + 1 < o->len) - PyList_Append(l, comma); - } - PyList_Append(l, suffix); - return PyUnicode_Join(sep, l); -} - -PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { - // TODO: Type check o - VecI64Object *v = (VecI64Object *)o; - if ((size_t)i < (size_t)v->len) { - return PyLong_FromLongLong(v->items[i]); - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } -} - -int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - // TODO: Type check o - VecI64Object *v = (VecI64Object *)self; - if ((size_t)i < (size_t)v->len) { - long long x = PyLong_AsLongLong(o); - if (x == -1 && PyErr_Occurred()) - return -1; - v->items[i] = x; - return 0; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return -1; - } -} - -Py_ssize_t vec_length(PyObject *o) { - // TODO: Type check o - return ((VecI64Object *)o)->len; -} - -static PyMappingMethods VecI64Mapping = { - .mp_length = vec_length, -}; - -static PySequenceMethods VecI64Sequence = { - .sq_item = vec_i64_get_item, - .sq_ass_item = vec_i64_ass_item, -}; - -static PyTypeObject VecI64Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecs.vec", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), - .tp_itemsize = sizeof(int64_t), - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = vec_i64_new, - .tp_free = PyObject_Del, - .tp_repr = (reprfunc)vec_i64_repr, - .tp_as_sequence = &VecI64Sequence, - .tp_as_mapping = &VecI64Mapping, -}; - -static VecI64Object * -vec_i64_alloc(Py_ssize_t size) -{ - VecI64Object *v; - /* TODO: Check for overflow */ - v = PyObject_NewVar(VecI64Object, &VecI64Type, size); - if (v == NULL) - return NULL; - return v; -} - -PyObject * -Vec_I64_New(Py_ssize_t size) -{ - VecI64Object *v; - v = vec_i64_alloc(size); - if (v == NULL) - return NULL; - for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = 0; - } - v->len = size; - return (PyObject *)v; -} - -PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {"", NULL}; - PyObject *t; - if (!PyArg_ParseTupleAndKeywords(args, kw, "O:vec", kwlist, &t)) { - return NULL; - } - return Vec_I64_New(0); -} - -PyObject * -Vec_Append(PyObject *obj, int64_t x) { - VecI64Object *vec = (VecI64Object *)obj; - Py_ssize_t cap = VEC_SIZE(vec); - Py_ssize_t len = vec->len; - if (len < cap) { - vec->items[len] = x; - vec->len = len + 1; - return (PyObject *)vec; - } else { - Py_ssize_t new_size = 2 * cap + 1; - VecI64Object *new = vec_i64_alloc(new_size); - if (new == NULL) - return NULL; - memcpy(new->items, vec->items, sizeof(int64_t) * len); - new->items[len] = x; - new->len = len + 1; - Py_DECREF(vec); - return (PyObject *)new; - } -} - -static PyObject * -vecs_append(PyObject *self, PyObject *args) +static PyObject *vecs_append(PyObject *self, PyObject *args) { PyObject *obj; int64_t x; @@ -149,7 +13,7 @@ vecs_append(PyObject *self, PyObject *args) // TODO: Type check obj Py_INCREF(obj); - return Vec_Append(obj, x); + return Vec_I64_Append(obj, x); } static PyMethodDef VecsMethods[] = { @@ -165,12 +29,6 @@ static PyModuleDef vecsmodule = { .m_methods = VecsMethods, }; -static VecI64Features I64Features = { - &VecI64Type, - Vec_I64_New, - Vec_Append, -}; - static VecCapsule Capsule = { &I64Features }; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index d9ccfcf4f7b40..bdb8fa3979ac4 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -52,4 +52,11 @@ typedef struct { // VecTFeatures *i64; } VecCapsule; +#define VEC_SIZE(v) ((v)->ob_base.ob_size) + +PyObject *Vec_I64_Append(PyObject *obj, int64_t x); + +extern PyTypeObject VecI64Type; +extern VecI64Features I64Features; + #endif From 3ab5d01ed3c196c47419f3596f9d313f2d0b369a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 11:47:19 +0000 Subject: [PATCH 014/208] Switch to vec[t]() for constructing an empty vec --- mypyc/lib-rt/vecs/vec_i64.c | 5 ++--- mypyc/lib-rt/vecs/vecs.c | 36 ++++++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index e7ea60bc64b7b..e8a37f57305bc 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -103,9 +103,8 @@ PyObject *Vec_I64_New(Py_ssize_t size) } PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {"", NULL}; - PyObject *t; - if (!PyArg_ParseTupleAndKeywords(args, kw, "O:vec", kwlist, &t)) { + static char *kwlist[] = {NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { return NULL; } return Vec_I64_New(0); diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 9df17126fde50..ee445917592c5 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -2,6 +2,32 @@ #include #include "vecs.h" +// Untyped vec +// cannot be instantiated, only used for isinstance and indexing vec[T] +typedef struct { + PyObject_HEAD +} VecGeneric; + +static PyObject *vec_class_getitem(PyObject *type, PyObject *item) +{ + Py_INCREF(&VecI64Type); + return (PyObject *)&VecI64Type; +} + +static PyMethodDef vec_methods[] = { + {"__class_getitem__", vec_class_getitem, METH_O|METH_CLASS, NULL}, + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + +PyTypeObject VecGenericType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec.generic", + .tp_basicsize = sizeof(VecGeneric), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, // XXX | Py_TPFLAGS_BASETYPE, + .tp_methods = vec_methods, +}; + static PyObject *vecs_append(PyObject *self, PyObject *args) { PyObject *obj; @@ -37,6 +63,8 @@ PyMODINIT_FUNC PyInit_vecs(void) { PyObject *m; + if (PyType_Ready(&VecGenericType) < 0) + return NULL; if (PyType_Ready(&VecI64Type) < 0) return NULL; @@ -44,9 +72,9 @@ PyInit_vecs(void) if (m == NULL) return NULL; - Py_INCREF(&VecI64Type); - if (PyModule_AddObject(m, "vec", (PyObject *) &VecI64Type) < 0) { - Py_DECREF(&VecI64Type); + Py_INCREF(&VecGenericType); + if (PyModule_AddObject(m, "vec", (PyObject *)&VecGenericType) < 0) { + Py_DECREF(&VecGenericType); Py_DECREF(m); return NULL; } @@ -57,7 +85,7 @@ PyInit_vecs(void) if (PyModule_AddObject(m, "_C_API", c_api) < 0) { Py_XDECREF(c_api); - Py_DECREF(&VecI64Type); + Py_DECREF(&VecGenericType); Py_DECREF(m); return NULL; } From e2136978749626c8ab2b7dadb92b35f3f3642e4d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 11:51:13 +0000 Subject: [PATCH 015/208] Update str() of type obj --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vecs.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index e8a37f57305bc..f8efeceac9511 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -67,7 +67,7 @@ static PySequenceMethods VecI64Sequence = { PyTypeObject VecI64Type = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecs.vec", + .tp_name = "vec[i64]", .tp_doc = "vec doc", .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), .tp_itemsize = sizeof(int64_t), diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index ee445917592c5..682db14f72139 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -21,7 +21,7 @@ static PyMethodDef vec_methods[] = { PyTypeObject VecGenericType = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec.generic", + .tp_name = "vec", .tp_basicsize = sizeof(VecGeneric), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, // XXX | Py_TPFLAGS_BASETYPE, From f62bf16c95d40eab4a4c4c6fe9ca689efaea9b68 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 12:01:31 +0000 Subject: [PATCH 016/208] Update repr of vec[i64] --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index f8efeceac9511..0ea7375886cea 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -7,7 +7,7 @@ static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); PyObject *vec_i64_repr(PyObject *self) { // TODO: Type check, refcounting, error handling VecI64Object *o = (VecI64Object *)self; - PyObject *prefix = Py_BuildValue("s", "vec(i64, ["); + PyObject *prefix = Py_BuildValue("s", "vec[i64](["); PyObject *suffix = Py_BuildValue("s", "])"); PyObject *l = Py_BuildValue("[]"); PyObject *sep = Py_BuildValue("s", ""); From 75f8803e412f5c224053c483ecc597409d868055 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 15:28:14 +0000 Subject: [PATCH 017/208] [WIP] Beginnings of support for vec[str] --- mypyc/lib-rt/vecs/vec_t.c | 101 ++++++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 94 ++++++++++++++++++++++++++++++++--- mypyc/lib-rt/vecs/vecs.h | 5 +- 3 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_t.c diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c new file mode 100644 index 0000000000000..4a2c3aff84f32 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -0,0 +1,101 @@ +#define PY_SSIZE_T_CLEAN +#include +#include "vecs.h" + +//static PyObject *vec_t_new(PyTypeObject *self, PyObject *args, PyObject *kw); + +PyObject *vec_t_repr(PyObject *self) { + // TODO: Type check, refcounting, error handling + VecTObject *o = (VecTObject *)self; + // TODO: Display actual type + PyObject *prefix = Py_BuildValue("s", "vec["); + PyObject *mid = Py_BuildValue("s", "](["); + PyObject *suffix = Py_BuildValue("s", "])"); + PyObject *l = Py_BuildValue("[]"); + PyObject *sep = Py_BuildValue("s", ""); + PyObject *comma = Py_BuildValue("s", ", "); + PyList_Append(l, prefix); + PyList_Append(l, PyObject_GetAttrString(o->item_type, "__name__")); + PyList_Append(l, mid); + for (int i = 0; i < o->len; i++) { + PyObject *r = PyObject_Repr(o->items[i]); + if (r == NULL) { + return NULL; + } + PyList_Append(l, r); + if (i + 1 < o->len) + PyList_Append(l, comma); + } + PyList_Append(l, suffix); + return PyUnicode_Join(sep, l); +} + +static int +VecT_traverse(VecTObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + Py_VISIT(self->items[i]); + } + return 0; +} + +static int +VecT_clear(VecTObject *self) +{ + Py_CLEAR(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + Py_CLEAR(self->items[i]); + } + return 0; +} + +static void +VecT_dealloc(VecTObject *self) +{ + PyObject_GC_UnTrack(self); + Py_TRASHCAN_BEGIN(self, VecT_dealloc) + VecT_clear(self); + //Py_DECREF(self->item_type); + //for (Py_ssize_t i = 0; i < self->len; i++) { + // Py_XDECREF(self->items[i]); + //} + //PyObject_GC_Del(self); + Py_TYPE(self)->tp_free((PyObject *)self); + Py_TRASHCAN_END +} + +PyTypeObject VecTType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec[t]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecTObject) - sizeof(PyObject *), + .tp_itemsize = sizeof(PyObject *), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = (traverseproc)VecT_traverse, + .tp_clear = (inquiry)VecT_clear, + .tp_dealloc = (destructor)VecT_dealloc, + //.tp_free = PyObject_GC_Del, + .tp_repr = (reprfunc)vec_t_repr, + //.tp_as_sequence = &VecI64Sequence, + //.tp_as_mapping = &VecI64Mapping, + // TODO: free +}; + +PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) +{ + VecTObject *v; + v = PyObject_GC_NewVar(VecTObject, &VecTType, size); + //v = VecTType.tp_alloc(&VecTType, size); + if (v == NULL) + return NULL; + + v->item_type = item_type; + v->len = size; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = NULL; + } + + PyObject_GC_Track(v); + return (PyObject *)v; +} diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 682db14f72139..bffc0bccffb3f 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -2,16 +2,80 @@ #include #include "vecs.h" + +static PyObject *i64_type_obj; + + +// vec type proxy +// +// Used for the result of generic vec[t] that must preserve knowledge of 't'. +// These aren't really types. + +// this can only instantiated +typedef struct { + PyObject_HEAD + PyObject *item_type; +} VecProxy; + +static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) +{ + static char *kwlist[] = {NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { + return NULL; + } + return Vec_T_New(0, ((VecProxy *)self)->item_type); +} + +static int +VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) +{ + Py_VISIT(self->item_type); + return 0; +} + +static void +VecProxy_dealloc(VecProxy *self) +{ + Py_CLEAR(self->item_type); + PyObject_GC_Del(self); +} + +PyTypeObject VecProxyType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec_proxy", + .tp_basicsize = sizeof(VecProxy), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_call = vec_proxy_call, + .tp_traverse = (traverseproc)VecProxy_traverse, + .tp_dealloc = (destructor)VecProxy_dealloc, +}; + + // Untyped vec -// cannot be instantiated, only used for isinstance and indexing vec[T] +// +// This cannot be instantiated, only used for isinstance and indexing: vec[T]. + typedef struct { PyObject_HEAD } VecGeneric; static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { - Py_INCREF(&VecI64Type); - return (PyObject *)&VecI64Type; + if (item == i64_type_obj) { + Py_INCREF(&VecI64Type); + return (PyObject *)&VecI64Type; + } else { + // TODO: Check validity + VecProxy *p; + p = PyObject_GC_New(VecProxy, &VecProxyType); + if (p == NULL) + return NULL; + Py_INCREF(item); + p->item_type = item; + PyObject_GC_Track(p); + return (PyObject *)p; + } } static PyMethodDef vec_methods[] = { @@ -24,10 +88,13 @@ PyTypeObject VecGenericType = { .tp_name = "vec", .tp_basicsize = sizeof(VecGeneric), .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, // XXX | Py_TPFLAGS_BASETYPE, + .tp_flags = Py_TPFLAGS_DEFAULT, .tp_methods = vec_methods, }; + +// Module-level functions + static PyObject *vecs_append(PyObject *self, PyObject *args) { PyObject *obj; @@ -62,13 +129,26 @@ static VecCapsule Capsule = { PyMODINIT_FUNC PyInit_vecs(void) { - PyObject *m; + PyObject *ext = PyImport_ImportModule("mypy_extensions"); + if (ext == NULL) { + return NULL; + } + + i64_type_obj = PyObject_GetAttrString(ext, "i64"); + if (i64_type_obj == NULL) { + return NULL; + } + if (PyType_Ready(&VecGenericType) < 0) return NULL; + if (PyType_Ready(&VecProxyType) < 0) + return NULL; + if (PyType_Ready(&VecTType) < 0) + return NULL; if (PyType_Ready(&VecI64Type) < 0) return NULL; - m = PyModule_Create(&vecsmodule); + PyObject *m = PyModule_Create(&vecsmodule); if (m == NULL) return NULL; @@ -90,5 +170,7 @@ PyInit_vecs(void) return NULL; } + Py_DECREF(ext); + return m; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index bdb8fa3979ac4..8774e16c65399 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -27,7 +27,7 @@ typedef struct _VecI64Features { typedef struct _VecTObject { PyObject_VAR_HEAD Py_ssize_t len; - PyObject *type_tagged; + PyObject *item_type; PyObject *items[1]; } VecTObject; @@ -54,9 +54,12 @@ typedef struct { #define VEC_SIZE(v) ((v)->ob_base.ob_size) +PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); + PyObject *Vec_I64_Append(PyObject *obj, int64_t x); extern PyTypeObject VecI64Type; +extern PyTypeObject VecTType; extern VecI64Features I64Features; #endif From efe565c75f04f1efb9181fe328001ce6eb4e4603 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 15:31:24 +0000 Subject: [PATCH 018/208] Implement len() for vec[str] --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 0ea7375886cea..a89f3192ebd16 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -51,7 +51,7 @@ int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -Py_ssize_t vec_length(PyObject *o) { +static Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o return ((VecI64Object *)o)->len; } diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 4a2c3aff84f32..6af3004a4692e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -65,9 +65,18 @@ VecT_dealloc(VecTObject *self) Py_TRASHCAN_END } +static Py_ssize_t vec_length(PyObject *o) { + // TODO: Type check o + return ((VecTObject *)o)->len; +} + +static PyMappingMethods VecTMapping = { + .mp_length = vec_length, +}; + PyTypeObject VecTType = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec[t]", + .tp_name = "vec", .tp_doc = "vec doc", .tp_basicsize = sizeof(VecTObject) - sizeof(PyObject *), .tp_itemsize = sizeof(PyObject *), @@ -78,7 +87,7 @@ PyTypeObject VecTType = { //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_t_repr, //.tp_as_sequence = &VecI64Sequence, - //.tp_as_mapping = &VecI64Mapping, + .tp_as_mapping = &VecTMapping, // TODO: free }; From 95ad1d56dcbcb339b84d275bd87f3224dd128add Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 16:00:48 +0000 Subject: [PATCH 019/208] Implement get item and append for vec[t] --- mypyc/lib-rt/vecs/vec_t.c | 49 +++++++++++++++++++++++++++++++++++---- mypyc/lib-rt/vecs/vecs.c | 34 ++++++++++++++++++++++----- mypyc/lib-rt/vecs/vecs.h | 11 ++++++++- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 6af3004a4692e..7c28e6d134ac6 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -30,6 +30,19 @@ PyObject *vec_t_repr(PyObject *self) { return PyUnicode_Join(sep, l); } +PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { + // TODO: Type check o + VecTObject *v = (VecTObject *)o; + if ((size_t)i < (size_t)v->len) { + PyObject *item = v->items[i]; + Py_INCREF(item); + return item; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -74,6 +87,11 @@ static PyMappingMethods VecTMapping = { .mp_length = vec_length, }; +static PySequenceMethods VecTSequence = { + .sq_item = vec_t_get_item, + //.sq_ass_item = vec_t_ass_item, +}; + PyTypeObject VecTType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", @@ -86,13 +104,12 @@ PyTypeObject VecTType = { .tp_dealloc = (destructor)VecT_dealloc, //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_t_repr, - //.tp_as_sequence = &VecI64Sequence, + .tp_as_sequence = &VecTSequence, .tp_as_mapping = &VecTMapping, // TODO: free }; -PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) -{ +VecTObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) { VecTObject *v; v = PyObject_GC_NewVar(VecTObject, &VecTType, size); //v = VecTType.tp_alloc(&VecTType, size); @@ -106,5 +123,29 @@ PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) } PyObject_GC_Track(v); - return (PyObject *)v; + return v; +} + +PyObject *Vec_T_Append(PyObject *obj, PyObject *x) { + VecTObject *vec = (VecTObject *)obj; + Py_ssize_t cap = VEC_SIZE(vec); + Py_ssize_t len = vec->len; + Py_INCREF(x); + if (len < cap) { + vec->items[len] = x; + vec->len = len + 1; + return (PyObject *)vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + // TODO: Avoid initializing to zero here + VecTObject *new = Vec_T_New(new_size, vec->item_type); + if (new == NULL) + return NULL; + memcpy(new->items, vec->items, sizeof(PyObject *) * len); + memset(vec->items, 0, sizeof(PyObject *) * len); + new->items[len] = x; + new->len = len + 1; + Py_DECREF(vec); + return (PyObject *)new; + } } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index bffc0bccffb3f..653a719669c3a 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -23,7 +23,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { return NULL; } - return Vec_T_New(0, ((VecProxy *)self)->item_type); + return (PyObject *)Vec_T_New(0, ((VecProxy *)self)->item_type); } static int @@ -97,16 +97,38 @@ PyTypeObject VecGenericType = { static PyObject *vecs_append(PyObject *self, PyObject *args) { - PyObject *obj; - int64_t x; + PyObject *vec; + PyObject *item; - if (!PyArg_ParseTuple(args, "OL", &obj, &x)) + if (!PyArg_ParseTuple(args, "OO", &vec, &item)) return NULL; // TODO: Type check obj - Py_INCREF(obj); - return Vec_I64_Append(obj, x); + if (VecI64_Check(vec)) { + // TODO: Check exact + int64_t x = PyLong_AsLong(item); + if (x == -1 && PyErr_Occurred()) { + return NULL; + } + Py_INCREF(vec); + return Vec_I64_Append(vec, x); + } else if (VecT_Check(vec)) { + VecTObject *v = (VecTObject *)vec; + int r = PyObject_IsInstance(item, v->item_type); + if (r > 0) { + Py_INCREF(vec); + return Vec_T_Append(vec, item); + } else if (r == -1) { + return NULL; + } else { + // TODO: exception + return NULL; + } + } else { + // TODO: exception + return NULL; + } } static PyMethodDef VecsMethods[] = { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 8774e16c65399..1d0d6b3fa9fa9 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -54,7 +54,8 @@ typedef struct { #define VEC_SIZE(v) ((v)->ob_base.ob_size) -PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); +VecTObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); +PyObject *Vec_T_Append(PyObject *obj, PyObject *x); PyObject *Vec_I64_Append(PyObject *obj, int64_t x); @@ -62,4 +63,12 @@ extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; extern VecI64Features I64Features; +inline int VecI64_Check(PyObject *o) { + return o->ob_type == &VecI64Type; +} + +inline int VecT_Check(PyObject *o) { + return o->ob_type == &VecTType; +} + #endif From 290534d5c66fcaba7608e642f816b4e9733a596a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 15 Nov 2021 16:14:39 +0000 Subject: [PATCH 020/208] Improvements to vec[t] support --- mypyc/lib-rt/vecs/vec_t.c | 16 +++++++++++++++- mypyc/lib-rt/vecs/vecs.c | 13 +++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 7c28e6d134ac6..bcdc40e93d7e1 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -43,6 +43,20 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } + +int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + // TODO: Type check o + VecTObject *v = (VecTObject *)self; + if ((size_t)i < (size_t)v->len) { + Py_INCREF(o); + v->items[i] = o; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -89,7 +103,7 @@ static PyMappingMethods VecTMapping = { static PySequenceMethods VecTSequence = { .sq_item = vec_t_get_item, - //.sq_ass_item = vec_t_ass_item, + .sq_ass_item = vec_t_ass_item, }; PyTypeObject VecTType = { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 653a719669c3a..f26c9d0fc5ffd 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -40,6 +40,18 @@ VecProxy_dealloc(VecProxy *self) PyObject_GC_Del(self); } +PyObject *VecProxy_repr(PyObject *self) { + // TODO: error handling, refcounting, etc. + PyObject *l = Py_BuildValue("[]"); + PyObject *prefix = Py_BuildValue("s", ""); + PyObject *sep = Py_BuildValue("s", ""); + PyList_Append(l, prefix); + PyList_Append(l, PyObject_GetAttrString(((VecProxy *)self)->item_type, "__name__")); + PyList_Append(l, suffix); + return PyUnicode_Join(sep, l); +} + PyTypeObject VecProxyType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec_proxy", @@ -49,6 +61,7 @@ PyTypeObject VecProxyType = { .tp_call = vec_proxy_call, .tp_traverse = (traverseproc)VecProxy_traverse, .tp_dealloc = (destructor)VecProxy_dealloc, + .tp_repr = (reprfunc)VecProxy_repr, }; From 80ea5fa228f258f995c5b860f12e7db06f64b01a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 17 Nov 2021 18:12:46 +0000 Subject: [PATCH 021/208] Improve runtime type checking --- mypyc/lib-rt/vecs/vec_i64.c | 3 ++- mypyc/lib-rt/vecs/vecs.c | 5 ++--- mypyc/lib-rt/vecs/vecs.h | 12 ++++++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index a89f3192ebd16..704ffb48e3abd 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -37,7 +37,8 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { } int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - // TODO: Type check o + if (check_float_error(o)) + return -1; VecI64Object *v = (VecI64Object *)self; if ((size_t)i < (size_t)v->len) { long long x = PyLong_AsLongLong(o); diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index f26c9d0fc5ffd..0937a742631e9 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -116,10 +116,9 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OO", &vec, &item)) return NULL; - // TODO: Type check obj - if (VecI64_Check(vec)) { - // TODO: Check exact + if (check_float_error(item)) + return NULL; int64_t x = PyLong_AsLong(item); if (x == -1 && PyErr_Occurred()) { return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 1d0d6b3fa9fa9..b7779f4810154 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -63,12 +63,20 @@ extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; extern VecI64Features I64Features; -inline int VecI64_Check(PyObject *o) { +static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } -inline int VecT_Check(PyObject *o) { +static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; } +static inline int check_float_error(PyObject *o) { + if (PyFloat_Check(o)) { + PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); + return 1; + } + return 0; +} + #endif From 073b0c3453fefa0f6b0915b97c588b62f439fc7b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 17 Nov 2021 20:26:35 +0000 Subject: [PATCH 022/208] Improve runtime error checking --- mypyc/lib-rt/vecs/vec_t.c | 14 ++++++++++++-- mypyc/lib-rt/vecs/vecs.c | 19 ++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index bcdc40e93d7e1..9547fbfbfaac4 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -31,7 +31,6 @@ PyObject *vec_t_repr(PyObject *self) { } PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { - // TODO: Type check o VecTObject *v = (VecTObject *)o; if ((size_t)i < (size_t)v->len) { PyObject *item = v->items[i]; @@ -45,8 +44,19 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - // TODO: Type check o VecTObject *v = (VecTObject *)self; + + int r = PyObject_IsInstance(o, v->item_type); + if (r <= 0) { + if (r == -1) { + return -1; + } else { + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return -1; + } + } + if ((size_t)i < (size_t)v->len) { Py_INCREF(o); v->items[i] = o; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 0937a742631e9..c57e09d06021c 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -65,9 +65,9 @@ PyTypeObject VecProxyType = { }; -// Untyped vec +// The 'vec' type // -// This cannot be instantiated, only used for isinstance and indexing: vec[T]. +// This cannot be instantiated, and it's only used for isinstance and indexing: vec[T]. typedef struct { PyObject_HEAD @@ -79,7 +79,15 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; } else { - // TODO: Check validity + PyTypeObject *t = item->ob_type; + if (t != &PyType_Type) { + PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); + return NULL; + } + if (item == &PyLong_Type || item == &PyFloat_Type || item == &PyBool_Type) { + PyErr_SetString(PyExc_ValueError, "unsupported type in vec[...]"); + return NULL; + } VecProxy *p; p = PyObject_GC_New(VecProxy, &VecProxyType); if (p == NULL) @@ -134,11 +142,12 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) } else if (r == -1) { return NULL; } else { - // TODO: exception + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); return NULL; } } else { - // TODO: exception + PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; } } From 67d9dd7e0cb8044ce3ff2c248f3b7f7f081cc462 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 09:16:44 +0000 Subject: [PATCH 023/208] Fix compiler warnings --- mypyc/lib-rt/vecs/vecs.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index c57e09d06021c..9f1887a55b89c 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -84,7 +84,9 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); return NULL; } - if (item == &PyLong_Type || item == &PyFloat_Type || item == &PyBool_Type) { + if (item == (PyObject *)&PyLong_Type + || item == (PyObject *)&PyFloat_Type + || item == (PyObject *)&PyBool_Type) { PyErr_SetString(PyExc_ValueError, "unsupported type in vec[...]"); return NULL; } From 308a8e80a4c38da5841bfe3ceaa5b4c583086278 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 09:41:04 +0000 Subject: [PATCH 024/208] Don't respect __instancecheck__ and allow metaclasses --- mypyc/lib-rt/vecs/vec_t.c | 18 +++++++----------- mypyc/lib-rt/vecs/vecs.c | 10 +++------- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 9547fbfbfaac4..e03ab0f6ad112 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -15,7 +15,7 @@ PyObject *vec_t_repr(PyObject *self) { PyObject *sep = Py_BuildValue("s", ""); PyObject *comma = Py_BuildValue("s", ", "); PyList_Append(l, prefix); - PyList_Append(l, PyObject_GetAttrString(o->item_type, "__name__")); + PyList_Append(l, PyObject_GetAttrString((PyObject *)o->item_type, "__name__")); PyList_Append(l, mid); for (int i = 0; i < o->len; i++) { PyObject *r = PyObject_Repr(o->items[i]); @@ -46,15 +46,11 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTObject *v = (VecTObject *)self; - int r = PyObject_IsInstance(o, v->item_type); - if (r <= 0) { - if (r == -1) { - return -1; - } else { - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); - return -1; - } + int r = PyObject_TypeCheck(o, v->item_type); + if (!r) { + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return -1; } if ((size_t)i < (size_t)v->len) { @@ -133,7 +129,7 @@ PyTypeObject VecTType = { // TODO: free }; -VecTObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) { +VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { VecTObject *v; v = PyObject_GC_NewVar(VecTObject, &VecTType, size); //v = VecTType.tp_alloc(&VecTType, size); diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 9f1887a55b89c..ef2e44cf94f1a 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -23,7 +23,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { return NULL; } - return (PyObject *)Vec_T_New(0, ((VecProxy *)self)->item_type); + return (PyObject *)Vec_T_New(0, (PyTypeObject *)((VecProxy *)self)->item_type); } static int @@ -79,8 +79,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; } else { - PyTypeObject *t = item->ob_type; - if (t != &PyType_Type) { + if (!PyObject_TypeCheck(item, &PyType_Type)) { PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); return NULL; } @@ -137,12 +136,9 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return Vec_I64_Append(vec, x); } else if (VecT_Check(vec)) { VecTObject *v = (VecTObject *)vec; - int r = PyObject_IsInstance(item, v->item_type); - if (r > 0) { + if (PyObject_TypeCheck(item, v->item_type)) { Py_INCREF(vec); return Vec_T_Append(vec, item); - } else if (r == -1) { - return NULL; } else { // TODO: better error message PyErr_SetString(PyExc_TypeError, "invalid item type"); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index b7779f4810154..483cf73fa8ac9 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -27,7 +27,7 @@ typedef struct _VecI64Features { typedef struct _VecTObject { PyObject_VAR_HEAD Py_ssize_t len; - PyObject *item_type; + PyTypeObject *item_type; PyObject *items[1]; } VecTObject; @@ -54,7 +54,7 @@ typedef struct { #define VEC_SIZE(v) ((v)->ob_base.ob_size) -VecTObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); +VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); PyObject *Vec_T_Append(PyObject *obj, PyObject *x); PyObject *Vec_I64_Append(PyObject *obj, int64_t x); From f2190fe2a9dcf3e6a3312767d284a73cd435bbb0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 10:05:37 +0000 Subject: [PATCH 025/208] Share vec item check code --- mypyc/lib-rt/vecs/vec_t.c | 8 +------- mypyc/lib-rt/vecs/vecs.c | 10 +++------- mypyc/lib-rt/vecs/vecs.h | 10 ++++++++++ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index e03ab0f6ad112..06b342606b9dd 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -45,14 +45,8 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTObject *v = (VecTObject *)self; - - int r = PyObject_TypeCheck(o, v->item_type); - if (!r) { - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); + if (!VecT_ItemCheck(v, o)) return -1; - } - if ((size_t)i < (size_t)v->len) { Py_INCREF(o); v->items[i] = o; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index ef2e44cf94f1a..cc88680d78319 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -136,14 +136,10 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return Vec_I64_Append(vec, x); } else if (VecT_Check(vec)) { VecTObject *v = (VecTObject *)vec; - if (PyObject_TypeCheck(item, v->item_type)) { - Py_INCREF(vec); - return Vec_T_Append(vec, item); - } else { - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); + if (!VecT_ItemCheck(v, item)) return NULL; - } + Py_INCREF(vec); + return Vec_T_Append(vec, item); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 483cf73fa8ac9..3fcae1bcdce35 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -79,4 +79,14 @@ static inline int check_float_error(PyObject *o) { return 0; } +static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { + if (PyObject_TypeCheck(item, v->item_type)) + return 1; + else { + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return 0; + } +} + #endif From 1b5cf18104b375ae2da9ace37881fc0d0f981628 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 11:46:34 +0000 Subject: [PATCH 026/208] Support vec[str | None] class proxy objects --- mypyc/lib-rt/vecs/vec_t.c | 8 +++++++ mypyc/lib-rt/vecs/vecs.c | 48 ++++++++++++++++++++++++++++++++++----- mypyc/lib-rt/vecs/vecs.h | 34 +++++++++++++++++++-------- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 06b342606b9dd..ea7b332842b31 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -1,3 +1,11 @@ +// Implementation of generic vec[t], when t is a plain type object. +// +// Examples of types supported: +// +// - vec[str] +// - vec[object] +// - vec[UserClass] + #define PY_SSIZE_T_CLEAN #include #include "vecs.h" diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index cc88680d78319..1265cc495c892 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -9,12 +9,12 @@ static PyObject *i64_type_obj; // vec type proxy // // Used for the result of generic vec[t] that must preserve knowledge of 't'. -// These aren't really types. - -// this can only instantiated +// These aren't really types. This only supports constructing instances. typedef struct { PyObject_HEAD PyObject *item_type; + int32_t depth; // Number of nested VecTExt or VecT types + int32_t optionals; // Flags for optional types on each nesting level } VecProxy; static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) @@ -47,7 +47,12 @@ PyObject *VecProxy_repr(PyObject *self) { PyObject *suffix = Py_BuildValue("s", "]'>"); PyObject *sep = Py_BuildValue("s", ""); PyList_Append(l, prefix); - PyList_Append(l, PyObject_GetAttrString(((VecProxy *)self)->item_type, "__name__")); + VecProxy *v = (VecProxy *)self; + PyList_Append(l, PyObject_GetAttrString(v->item_type, "__name__")); + if (v->optionals) { + PyObject *none = Py_BuildValue("s", " | None"); + PyList_Append(l, none); + } PyList_Append(l, suffix); return PyUnicode_Join(sep, l); } @@ -73,15 +78,42 @@ typedef struct { PyObject_HEAD } VecGeneric; +static PyObject *extract_optional_item(PyObject *item) { + PyObject *args = PyObject_GetAttrString(item, "__args__"); + if (args == NULL) + goto error; + if (!PyTuple_CheckExact(args)) + goto error; + if (PyTuple_GET_SIZE(args) != 2) + goto error; + PyObject *item0 = PyTuple_GET_ITEM(args, 0); + PyObject *item1 = PyTuple_GET_ITEM(args, 1); + if (item0 == (PyObject *)Py_None->ob_type) { + Py_DECREF(args); + return item1; + } else if (item1 == (PyObject *)Py_None->ob_type) { + Py_DECREF(args); + return item0; + } + error: + Py_DECREF(args); + return NULL; +} + static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { if (item == i64_type_obj) { Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; } else { + int32_t optionals = 0; if (!PyObject_TypeCheck(item, &PyType_Type)) { - PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); - return NULL; + item = extract_optional_item(item); + if (item == NULL) { + PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); + return NULL; + } + optionals = 1; } if (item == (PyObject *)&PyLong_Type || item == (PyObject *)&PyFloat_Type @@ -95,6 +127,8 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) return NULL; Py_INCREF(item); p->item_type = item; + p->depth = 0; + p->optionals = optionals; PyObject_GC_Track(p); return (PyObject *)p; } @@ -182,6 +216,8 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecTType) < 0) return NULL; + if (PyType_Ready(&VecTExtType) < 0) + return NULL; if (PyType_Ready(&VecI64Type) < 0) return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 3fcae1bcdce35..84a5158a7de5c 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -4,12 +4,31 @@ #define PY_SSIZE_T_CLEAN #include +// vec[i64] typedef struct _VecI64Object { PyObject_VAR_HEAD Py_ssize_t len; int64_t items[1]; } VecI64Object; +// Simple generic vec: vec[t] when t is a type object +typedef struct _VecTObject { + PyObject_VAR_HEAD + Py_ssize_t len; + PyTypeObject *item_type; + PyObject *items[1]; +} VecTObject; + +// Extended generic vec type: vec[t | None], vec[vec[...]], etc. +typedef struct _VecTExtObject { + PyObject_VAR_HEAD + Py_ssize_t len; + PyTypeObject *item_type; + int32_t depth; // Number of nested VecTExt or VecT types + int32_t optionals; // Flags for optional types on each nesting level + PyObject *items[1]; +} VecTExtObject; + // vec[i64] operations + type object (stored in a capsule) typedef struct _VecI64Features { PyTypeObject *type; @@ -24,13 +43,6 @@ typedef struct _VecI64Features { // iter? } VecI64Features; -typedef struct _VecTObject { - PyObject_VAR_HEAD - Py_ssize_t len; - PyTypeObject *item_type; - PyObject *items[1]; -} VecTObject; - // vec[T] operations + type object (stored in a capsule) // // T can be a class type, T | None or vec[T] (or a combination of these) @@ -49,18 +61,20 @@ typedef struct _VecTFeatures { typedef struct { VecI64Features *i64; - // VecTFeatures *i64; + // VecTFeatures *t; + // VecTExtFeatures *t_ext; } VecCapsule; #define VEC_SIZE(v) ((v)->ob_base.ob_size) +PyObject *Vec_I64_Append(PyObject *obj, int64_t x); + VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); PyObject *Vec_T_Append(PyObject *obj, PyObject *x); -PyObject *Vec_I64_Append(PyObject *obj, int64_t x); - extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; +extern PyTypeObject VecTExtType; extern VecI64Features I64Features; static inline int VecI64_Check(PyObject *o) { From da92666a93a679be09c70891f93ab9d82ed855a0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 12:46:34 +0000 Subject: [PATCH 027/208] Start implementing real vec[t | None] support --- mypyc/lib-rt/vecs/vec_t_ext.c | 149 ++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 15 +++- mypyc/lib-rt/vecs/vecs.h | 50 +++++++++--- 3 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_t_ext.c diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c new file mode 100644 index 0000000000000..7d60df1acda0b --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -0,0 +1,149 @@ +// Implementation of generic vec[t], when t is an optional type or nested generic vec. +// +// Examples of types supported: +// - vec[str | None] +// - vec[vec[str]] +// - vec[vec[str | None] | None] + +#define PY_SSIZE_T_CLEAN +#include +#include "vecs.h" + +PyObject *vec_t_ext_repr(PyObject *self) { + // TODO: Type check, refcounting, error handling + VecTExtObject *o = (VecTExtObject *)self; + // TODO: Display actual type + PyObject *prefix = Py_BuildValue("s", "vec["); + // TODO: Don't hard code + PyObject *mid = Py_BuildValue("s", " | None](["); + PyObject *suffix = Py_BuildValue("s", "])"); + PyObject *l = Py_BuildValue("[]"); + PyObject *sep = Py_BuildValue("s", ""); + PyObject *comma = Py_BuildValue("s", ", "); + PyList_Append(l, prefix); + PyList_Append(l, PyObject_GetAttrString((PyObject *)o->item_type, "__name__")); + PyList_Append(l, mid); + for (int i = 0; i < o->len; i++) { + PyObject *r = PyObject_Repr(o->items[i]); + if (r == NULL) { + return NULL; + } + PyList_Append(l, r); + if (i + 1 < o->len) + PyList_Append(l, comma); + } + PyList_Append(l, suffix); + return PyUnicode_Join(sep, l); +} + +static int +VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + // TODO: NULL checks? + Py_VISIT(self->items[i]); + } + return 0; +} + +static int +VecTExt_clear(VecTExtObject *self) +{ + Py_CLEAR(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + // TODO: NULL checks? + Py_CLEAR(self->items[i]); + } + return 0; +} + +static void +VecTExt_dealloc(VecTExtObject *self) +{ + PyObject_GC_UnTrack(self); + Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) + VecTExt_clear(self); + //Py_DECREF(self->item_type); + //for (Py_ssize_t i = 0; i < self->len; i++) { + // Py_XDECREF(self->items[i]); + //} + //PyObject_GC_Del(self); + Py_TYPE(self)->tp_free((PyObject *)self); + Py_TRASHCAN_END +} + +static Py_ssize_t vec_ext_length(PyObject *o) { + // TODO: Type check o + return ((VecTExtObject *)o)->len; +} + +static PyMappingMethods VecTExtMapping = { + .mp_length = vec_ext_length, +}; + +//static PySequenceMethods VecTSequence = { +// .sq_item = vec_t_get_item, +// .sq_ass_item = vec_t_ass_item, +//}; + +PyTypeObject VecTExtType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecTExtObject) - sizeof(PyObject *), + .tp_itemsize = sizeof(PyObject *), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = (traverseproc)VecTExt_traverse, + .tp_clear = (inquiry)VecTExt_clear, + .tp_dealloc = (destructor)VecTExt_dealloc, + //.tp_free = PyObject_GC_Del, + .tp_repr = (reprfunc)vec_t_ext_repr, + //.tp_as_sequence = &VecTExtSequence, + .tp_as_mapping = &VecTExtMapping, + // TODO: free +}; + +VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, + int32_t depth) { + VecTExtObject *v; + v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); + //v = VecTType.tp_alloc(&VecTType, size); + if (v == NULL) + return NULL; + + v->item_type = item_type; + v->len = size; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = NULL; + } + v->optionals = optionals; + v->depth = depth; + + PyObject_GC_Track(v); + return v; +} + +PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { + VecTExtObject *vec = (VecTExtObject *)obj; + Py_ssize_t cap = VEC_SIZE(vec); + Py_ssize_t len = vec->len; + Py_INCREF(x); + if (len < cap) { + vec->items[len] = x; + vec->len = len + 1; + return (PyObject *)vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + // TODO: Avoid initializing to zero here + VecTExtObject *new = Vec_T_Ext_New(new_size, vec->item_type, vec->optionals, vec->depth); + if (new == NULL) + return NULL; + memcpy(new->items, vec->items, sizeof(PyObject *) * len); + memset(vec->items, 0, sizeof(PyObject *) * len); + new->items[len] = x; + new->len = len + 1; + Py_DECREF(vec); + return (PyObject *)new; + } +} diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 1265cc495c892..afd11a933de28 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -23,7 +23,12 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { return NULL; } - return (PyObject *)Vec_T_New(0, (PyTypeObject *)((VecProxy *)self)->item_type); + VecProxy *p = (VecProxy *)self; + if (p->optionals == 0 && p->depth == 0) { + return (PyObject *)Vec_T_New(0, (PyTypeObject *)p->item_type); + } else { + return (PyObject *)Vec_T_Ext_New(0, (PyTypeObject *)p->item_type, p->optionals, p->depth); + } } static int @@ -81,7 +86,7 @@ typedef struct { static PyObject *extract_optional_item(PyObject *item) { PyObject *args = PyObject_GetAttrString(item, "__args__"); if (args == NULL) - goto error; + return NULL; if (!PyTuple_CheckExact(args)) goto error; if (PyTuple_GET_SIZE(args) != 2) @@ -174,6 +179,12 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return NULL; Py_INCREF(vec); return Vec_T_Append(vec, item); + } else if (VecTExt_Check(vec)) { + VecTExtObject *v = (VecTExtObject *)vec; + if (!VecTExt_ItemCheck(v, item)) + return NULL; + Py_INCREF(vec); + return Vec_T_Ext_Append(vec, item); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 84a5158a7de5c..3f36937ed1215 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -67,40 +67,70 @@ typedef struct { #define VEC_SIZE(v) ((v)->ob_base.ob_size) -PyObject *Vec_I64_Append(PyObject *obj, int64_t x); - -VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); -PyObject *Vec_T_Append(PyObject *obj, PyObject *x); +// Type objects extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; extern PyTypeObject VecTExtType; extern VecI64Features I64Features; +// vec[i64] operations + static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } +PyObject *Vec_I64_Append(PyObject *obj, int64_t x); + +// vec[t] operations (simple) + static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; } -static inline int check_float_error(PyObject *o) { - if (PyFloat_Check(o)) { - PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); +static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { + if (PyObject_TypeCheck(item, v->item_type)) return 1; + else { + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return 0; } - return 0; } -static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { +VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); +PyObject *Vec_T_Append(PyObject *obj, PyObject *x); + +// vec[t] operations (extended) + +static inline int VecTExt_Check(PyObject *o) { + return o->ob_type == &VecTExtType; +} + +static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *item) { if (PyObject_TypeCheck(item, v->item_type)) return 1; - else { + else if (item == Py_None && (v->optionals & 1)) { + return 1; + } else { // TODO: better error message PyErr_SetString(PyExc_TypeError, "invalid item type"); return 0; } } +VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, + int32_t depth); +PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x); + +// Misc helpers + +static inline int check_float_error(PyObject *o) { + if (PyFloat_Check(o)) { + PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); + return 1; + } + return 0; +} + #endif From cb7a7ac498ed88dad5dca2df53f76c259bbceb26 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 13:56:45 +0000 Subject: [PATCH 028/208] Support proxy types with nested vecs --- mypyc/lib-rt/vecs/vecs.c | 44 +++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index afd11a933de28..8fa8c40c9b036 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -45,19 +45,28 @@ VecProxy_dealloc(VecProxy *self) PyObject_GC_Del(self); } +PyObject *vec_type_to_str(PyObject *item_type, int32_t depth, int32_t optionals) { + PyObject *item; + if (depth == 0) + item = PyObject_GetAttrString(item_type, "__name__"); + else + item = vec_type_to_str(item_type, depth - 1, optionals >> 1); + + if (optionals & 1) { + item = PyUnicode_FromFormat("%U | None", item); + } + return PyUnicode_FromFormat("vec[%U]", item); +} + PyObject *VecProxy_repr(PyObject *self) { // TODO: error handling, refcounting, etc. PyObject *l = Py_BuildValue("[]"); - PyObject *prefix = Py_BuildValue("s", ""); + PyObject *prefix = Py_BuildValue("s", ""); PyObject *sep = Py_BuildValue("s", ""); PyList_Append(l, prefix); VecProxy *v = (VecProxy *)self; - PyList_Append(l, PyObject_GetAttrString(v->item_type, "__name__")); - if (v->optionals) { - PyObject *none = Py_BuildValue("s", " | None"); - PyList_Append(l, none); - } + PyList_Append(l, vec_type_to_str(v->item_type, v->depth, v->optionals)); PyList_Append(l, suffix); return PyUnicode_Join(sep, l); } @@ -85,8 +94,10 @@ typedef struct { static PyObject *extract_optional_item(PyObject *item) { PyObject *args = PyObject_GetAttrString(item, "__args__"); - if (args == NULL) + if (args == NULL) { + PyErr_Clear(); return NULL; + } if (!PyTuple_CheckExact(args)) goto error; if (PyTuple_GET_SIZE(args) != 2) @@ -112,13 +123,22 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) return (PyObject *)&VecI64Type; } else { int32_t optionals = 0; + int32_t depth = 0; if (!PyObject_TypeCheck(item, &PyType_Type)) { - item = extract_optional_item(item); - if (item == NULL) { + PyObject *it = extract_optional_item(item); + if (it != NULL) { + optionals = 1; + item = it; + } + if (item->ob_type == &VecProxyType) { + VecProxy *p = (VecProxy *)item; + item = p->item_type; + depth = p->depth + 1; + optionals |= p->optionals << 1; + } else if (!PyObject_TypeCheck(item, &PyType_Type)) { PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); return NULL; } - optionals = 1; } if (item == (PyObject *)&PyLong_Type || item == (PyObject *)&PyFloat_Type @@ -132,7 +152,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) return NULL; Py_INCREF(item); p->item_type = item; - p->depth = 0; + p->depth = depth; p->optionals = optionals; PyObject_GC_Track(p); return (PyObject *)p; From 37fd0a64abaef6845aaf04661fc67a669d7642a8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 16:55:32 +0000 Subject: [PATCH 029/208] Improvements to optional/nested vecs --- mypyc/lib-rt/vecs/vec_t_ext.c | 46 +++++++++++++------------------ mypyc/lib-rt/vecs/vecs.c | 51 +++++++++++++++++++++++++++++++---- mypyc/lib-rt/vecs/vecs.h | 18 ++++++++++--- 3 files changed, 79 insertions(+), 36 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 7d60df1acda0b..e3a8a7c60226c 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -10,30 +10,20 @@ #include "vecs.h" PyObject *vec_t_ext_repr(PyObject *self) { - // TODO: Type check, refcounting, error handling - VecTExtObject *o = (VecTExtObject *)self; - // TODO: Display actual type - PyObject *prefix = Py_BuildValue("s", "vec["); - // TODO: Don't hard code - PyObject *mid = Py_BuildValue("s", " | None](["); - PyObject *suffix = Py_BuildValue("s", "])"); - PyObject *l = Py_BuildValue("[]"); - PyObject *sep = Py_BuildValue("s", ""); - PyObject *comma = Py_BuildValue("s", ", "); - PyList_Append(l, prefix); - PyList_Append(l, PyObject_GetAttrString((PyObject *)o->item_type, "__name__")); - PyList_Append(l, mid); - for (int i = 0; i < o->len; i++) { - PyObject *r = PyObject_Repr(o->items[i]); - if (r == NULL) { - return NULL; - } - PyList_Append(l, r); - if (i + 1 < o->len) - PyList_Append(l, comma); + VecTExtObject *v = (VecTExtObject *)self; + return vec_repr(self, v->item_type, v->depth, v->optionals, 1); +} + +PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { + VecTExtObject *v = (VecTExtObject *)o; + if ((size_t)i < (size_t)v->len) { + PyObject *item = v->items[i]; + Py_INCREF(item); + return item; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; } - PyList_Append(l, suffix); - return PyUnicode_Join(sep, l); } static int @@ -82,10 +72,10 @@ static PyMappingMethods VecTExtMapping = { .mp_length = vec_ext_length, }; -//static PySequenceMethods VecTSequence = { -// .sq_item = vec_t_get_item, -// .sq_ass_item = vec_t_ass_item, -//}; +static PySequenceMethods VecTExtSequence = { + .sq_item = vec_t_ext_get_item, +// .sq_ass_item = vec_t_ext_ass_item, +}; PyTypeObject VecTExtType = { PyVarObject_HEAD_INIT(NULL, 0) @@ -99,7 +89,7 @@ PyTypeObject VecTExtType = { .tp_dealloc = (destructor)VecTExt_dealloc, //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_t_ext_repr, - //.tp_as_sequence = &VecTExtSequence, + .tp_as_sequence = &VecTExtSequence, .tp_as_mapping = &VecTExtMapping, // TODO: free }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 8fa8c40c9b036..8cfacaeb76a3f 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -12,7 +12,7 @@ static PyObject *i64_type_obj; // These aren't really types. This only supports constructing instances. typedef struct { PyObject_HEAD - PyObject *item_type; + PyTypeObject *item_type; int32_t depth; // Number of nested VecTExt or VecT types int32_t optionals; // Flags for optional types on each nesting level } VecProxy; @@ -45,10 +45,10 @@ VecProxy_dealloc(VecProxy *self) PyObject_GC_Del(self); } -PyObject *vec_type_to_str(PyObject *item_type, int32_t depth, int32_t optionals) { +PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals) { PyObject *item; if (depth == 0) - item = PyObject_GetAttrString(item_type, "__name__"); + item = PyObject_GetAttrString((PyObject *)item_type, "__name__"); else item = vec_type_to_str(item_type, depth - 1, optionals >> 1); @@ -132,7 +132,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } if (item->ob_type == &VecProxyType) { VecProxy *p = (VecProxy *)item; - item = p->item_type; + item = (PyObject *)p->item_type; depth = p->depth + 1; optionals |= p->optionals << 1; } else if (!PyObject_TypeCheck(item, &PyType_Type)) { @@ -151,7 +151,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (p == NULL) return NULL; Py_INCREF(item); - p->item_type = item; + p->item_type = (PyTypeObject *)item; p->depth = depth; p->optionals = optionals; PyObject_GC_Track(p); @@ -173,6 +173,47 @@ PyTypeObject VecGenericType = { .tp_methods = vec_methods, }; +PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, + int verbose) { + PyObject *l = Py_BuildValue("[]"); + PyObject *prefix; + PyObject *mid; + PyObject *suffix; + PyObject *sep = Py_BuildValue("s", ""); + PyObject *comma = Py_BuildValue("s", ", "); + if (verbose) { + prefix = vec_type_to_str(item_type, depth, optionals); + mid = Py_BuildValue("s", "(["); + suffix = Py_BuildValue("s", "])"); + } else { + prefix = Py_BuildValue("s", ""); + mid = Py_BuildValue("s", "["); + suffix = Py_BuildValue("s", "]"); + } + PyList_Append(l, prefix); + PyList_Append(l, mid); + + Py_ssize_t len = PyObject_Length(vec); + + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *it = PySequence_GetItem(vec, i); + PyObject *r; + if (depth == 0) { + r = PyObject_Repr(it); + } else { + r = vec_repr(it, item_type, depth - 1, optionals >> 1, 0); + } + if (r == NULL) + return NULL; + PyList_Append(l, r); + if (i + 1 < len) + PyList_Append(l, comma); + } + + PyList_Append(l, suffix); + + return PyUnicode_Join(sep, l); +} // Module-level functions diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 3f36937ed1215..d234eed914447 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -107,10 +107,18 @@ static inline int VecTExt_Check(PyObject *o) { return o->ob_type == &VecTExtType; } -static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *item) { - if (PyObject_TypeCheck(item, v->item_type)) +static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { + if (v->depth == 0 && PyObject_TypeCheck(it, v->item_type)) { + return 1; + } else if (it == Py_None && (v->optionals & 1)) { return 1; - else if (item == Py_None && (v->optionals & 1)) { + } else if (v->depth == 1 && it->ob_type == &VecTType + && ((VecTExtObject *)it)->item_type == v->item_type) { + return 1; + } else if (it->ob_type == &VecTExtType + && ((VecTExtObject *)it)->depth == v->depth - 1 + && ((VecTExtObject *)it)->item_type == v->item_type + && ((VecTExtObject *)it)->optionals == (v->optionals >> 1)) { return 1; } else { // TODO: better error message @@ -133,4 +141,8 @@ static inline int check_float_error(PyObject *o) { return 0; } +PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals); +PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, + int verbose); + #endif From afc2dbdd8ecc6d4334fd51d00167fffdd6c69536 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 16:56:52 +0000 Subject: [PATCH 030/208] Refactor --- mypyc/lib-rt/vecs/vec_t.c | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index ea7b332842b31..5efd18c277113 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -13,29 +13,8 @@ //static PyObject *vec_t_new(PyTypeObject *self, PyObject *args, PyObject *kw); PyObject *vec_t_repr(PyObject *self) { - // TODO: Type check, refcounting, error handling - VecTObject *o = (VecTObject *)self; - // TODO: Display actual type - PyObject *prefix = Py_BuildValue("s", "vec["); - PyObject *mid = Py_BuildValue("s", "](["); - PyObject *suffix = Py_BuildValue("s", "])"); - PyObject *l = Py_BuildValue("[]"); - PyObject *sep = Py_BuildValue("s", ""); - PyObject *comma = Py_BuildValue("s", ", "); - PyList_Append(l, prefix); - PyList_Append(l, PyObject_GetAttrString((PyObject *)o->item_type, "__name__")); - PyList_Append(l, mid); - for (int i = 0; i < o->len; i++) { - PyObject *r = PyObject_Repr(o->items[i]); - if (r == NULL) { - return NULL; - } - PyList_Append(l, r); - if (i + 1 < o->len) - PyList_Append(l, comma); - } - PyList_Append(l, suffix); - return PyUnicode_Join(sep, l); + VecTObject *v = (VecTObject *)self; + return vec_repr(self, v->item_type, 0, 0, 1); } PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { From d9733fa74c0ed18a70c90bb11d96d84a5f8c3710 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 17:00:53 +0000 Subject: [PATCH 031/208] Refactor --- mypyc/lib-rt/vecs/vec_i64.c | 19 +------------------ mypyc/lib-rt/vecs/vecs.c | 9 +++++---- mypyc/lib-rt/vecs/vecs.h | 1 + 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 704ffb48e3abd..d7d7f3fb45918 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -5,24 +5,7 @@ static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); PyObject *vec_i64_repr(PyObject *self) { - // TODO: Type check, refcounting, error handling - VecI64Object *o = (VecI64Object *)self; - PyObject *prefix = Py_BuildValue("s", "vec[i64](["); - PyObject *suffix = Py_BuildValue("s", "])"); - PyObject *l = Py_BuildValue("[]"); - PyObject *sep = Py_BuildValue("s", ""); - PyObject *comma = Py_BuildValue("s", ", "); - PyList_Append(l, prefix); - for (int i = 0; i < o->len; i++) { - char s[100]; - sprintf(s, "%lld", (long long)o->items[i]); - PyObject *x = Py_BuildValue("s", s); - PyList_Append(l, x); - if (i + 1 < o->len) - PyList_Append(l, comma); - } - PyList_Append(l, suffix); - return PyUnicode_Join(sep, l); + return vec_repr(self, I64TypeObj, 0, 0, 1); } PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 8cfacaeb76a3f..9e0d5060ba1c6 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -3,7 +3,7 @@ #include "vecs.h" -static PyObject *i64_type_obj; +PyTypeObject *I64TypeObj; // vec type proxy @@ -118,7 +118,7 @@ static PyObject *extract_optional_item(PyObject *item) { static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { - if (item == i64_type_obj) { + if (item == (PyObject *)I64TypeObj) { Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; } else { @@ -277,8 +277,9 @@ PyInit_vecs(void) return NULL; } - i64_type_obj = PyObject_GetAttrString(ext, "i64"); - if (i64_type_obj == NULL) { + I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); + // TODO: Check that it's a type object! + if (I64TypeObj == NULL) { return NULL; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index d234eed914447..0a1a69a1c9f31 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -73,6 +73,7 @@ extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; extern PyTypeObject VecTExtType; extern VecI64Features I64Features; +extern PyTypeObject *I64TypeObj; // vec[i64] operations From 45d1b1e6ea07ee7594e4af0439bb6ffe0f42a352 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 20 Nov 2021 17:21:38 +0000 Subject: [PATCH 032/208] Add set item op --- mypyc/lib-rt/vecs/vec_t_ext.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index e3a8a7c60226c..2f40ea0b2dc21 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -26,6 +26,20 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { } } +int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + VecTExtObject *v = (VecTExtObject *)self; + if (!VecTExt_ItemCheck(v, o)) + return -1; + if ((size_t)i < (size_t)v->len) { + Py_INCREF(o); + v->items[i] = o; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + static int VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { @@ -74,7 +88,7 @@ static PyMappingMethods VecTExtMapping = { static PySequenceMethods VecTExtSequence = { .sq_item = vec_t_ext_get_item, -// .sq_ass_item = vec_t_ext_ass_item, + .sq_ass_item = vec_t_ext_ass_item, }; PyTypeObject VecTExtType = { From b9bc3ec10982e01cbc3f56b508df7907b9e992e5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Nov 2021 10:30:43 +0000 Subject: [PATCH 033/208] Implement vec[i64](init) --- mypyc/lib-rt/vecs/vec_i64.c | 34 +++++++++++++++++++++++++++++++--- mypyc/lib-rt/vecs/vec_t.c | 1 - 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index d7d7f3fb45918..f5a9975d6bfe8 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -86,12 +86,40 @@ PyObject *Vec_I64_New(Py_ssize_t size) return (PyObject *)v; } +VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { + VecI64Object *v = vec_i64_alloc(0); + if (v == NULL) + return NULL; + v->len = 0; + + PyObject *iter = PyObject_GetIter(iterable); + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + int64_t x = PyLong_AsLongLong(item); + if (x == -1 && PyErr_Occurred()) + return NULL; + v = (VecI64Object *)Vec_I64_Append((PyObject *)v, x); + if (v == NULL) + return NULL; + Py_DECREF(item); + } + Py_DECREF(iter); + if (PyErr_Occurred()) + return NULL; + return v; +} + PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { + static char *kwlist[] = {"", NULL}; + PyObject *init = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { return NULL; } - return Vec_I64_New(0); + if (init == NULL) { + return Vec_I64_New(0); + } else { + return (PyObject *)Vec_I64_FromIterable(init); + } } PyObject *Vec_I64_Append(PyObject *obj, int64_t x) { diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 5efd18c277113..0b17f32524c7c 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -29,7 +29,6 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } - int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTObject *v = (VecTObject *)self; if (!VecT_ItemCheck(v, o)) From 91fd2a213c993821be6b08c47deb71607f41e5a5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Nov 2021 11:09:37 +0000 Subject: [PATCH 034/208] Initialize vec[t] from iterable --- mypyc/lib-rt/vecs/vec_i64.c | 20 ++++++++++++++++---- mypyc/lib-rt/vecs/vec_t.c | 36 ++++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 11 ++++++++--- mypyc/lib-rt/vecs/vecs.h | 1 + 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index f5a9975d6bfe8..1be708a4354f3 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -93,19 +93,31 @@ VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { v->len = 0; PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + Py_DECREF(v); + return NULL; + } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { int64_t x = PyLong_AsLongLong(item); - if (x == -1 && PyErr_Occurred()) + Py_DECREF(item); + if (x == -1 && PyErr_Occurred()) { + Py_DECREF(iter); + Py_DECREF(v); return NULL; + } v = (VecI64Object *)Vec_I64_Append((PyObject *)v, x); - if (v == NULL) + if (v == NULL) { + Py_DECREF(iter); + Py_DECREF(v); return NULL; - Py_DECREF(item); + } } Py_DECREF(iter); - if (PyErr_Occurred()) + if (PyErr_Occurred()) { + Py_DECREF(v); return NULL; + } return v; } diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 0b17f32524c7c..e2f342b5cd062 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -126,6 +126,42 @@ VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { return v; } +VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { + VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, 0); + if (v == NULL) + return NULL; + v->len = 0; + v->item_type = item_type; + + PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + Py_DECREF(v); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + if (!VecT_ItemCheck(v, item)) { + Py_DECREF(iter); + Py_DECREF(v); + Py_DECREF(item); + return NULL; + } + v = (VecTObject *)Vec_T_Append((PyObject *)v, item); + Py_DECREF(item); + if (v == NULL) { + Py_DECREF(iter); + Py_DECREF(v); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + return v; +} + PyObject *Vec_T_Append(PyObject *obj, PyObject *x) { VecTObject *vec = (VecTObject *)obj; Py_ssize_t cap = VEC_SIZE(vec); diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 9e0d5060ba1c6..b71a1d665635b 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -19,13 +19,18 @@ typedef struct { static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {NULL}; - if (!PyArg_ParseTupleAndKeywords(args, kw, ":vec", kwlist)) { + static char *kwlist[] = {"", NULL}; + PyObject *init = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { return NULL; } VecProxy *p = (VecProxy *)self; if (p->optionals == 0 && p->depth == 0) { - return (PyObject *)Vec_T_New(0, (PyTypeObject *)p->item_type); + if (init == NULL) { + return (PyObject *)Vec_T_New(0, (PyTypeObject *)p->item_type); + } else { + return (PyObject *)Vec_T_FromIterable((PyTypeObject *)p->item_type, init); + } } else { return (PyObject *)Vec_T_Ext_New(0, (PyTypeObject *)p->item_type, p->optionals, p->depth); } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 0a1a69a1c9f31..91bd2f89ece0b 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -100,6 +100,7 @@ static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { } VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); +VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable); PyObject *Vec_T_Append(PyObject *obj, PyObject *x); // vec[t] operations (extended) From f5cd66061583c84c69f9bb765fe2d0baea769b13 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 21 Nov 2021 15:18:53 +0000 Subject: [PATCH 035/208] More initialization from iterable --- mypyc/lib-rt/vecs/vec_t_ext.c | 39 +++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 8 ++++++- mypyc/lib-rt/vecs/vecs.h | 3 ++- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 2f40ea0b2dc21..4325b7a1b8493 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -128,6 +128,45 @@ VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t o return v; } +VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, + PyObject *iterable) { + VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, 0); + if (v == NULL) + return NULL; + v->len = 0; + v->item_type = item_type; + v->optionals = optionals; + v->depth = depth; + + PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + Py_DECREF(v); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + if (!VecTExt_ItemCheck(v, item)) { + Py_DECREF(iter); + Py_DECREF(v); + Py_DECREF(item); + return NULL; + } + v = (VecTExtObject *)Vec_T_Ext_Append((PyObject *)v, item); + Py_DECREF(item); + if (v == NULL) { + Py_DECREF(iter); + Py_DECREF(v); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + return v; +} + PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { VecTExtObject *vec = (VecTExtObject *)obj; Py_ssize_t cap = VEC_SIZE(vec); diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index b71a1d665635b..e163944a05211 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -32,7 +32,13 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) return (PyObject *)Vec_T_FromIterable((PyTypeObject *)p->item_type, init); } } else { - return (PyObject *)Vec_T_Ext_New(0, (PyTypeObject *)p->item_type, p->optionals, p->depth); + if (init == NULL) { + return (PyObject *)Vec_T_Ext_New(0, (PyTypeObject *)p->item_type, p->optionals, + p->depth); + } else { + return (PyObject *)Vec_T_Ext_FromIterable((PyTypeObject *)p->item_type, p->optionals, + p->depth, init); + } } } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 91bd2f89ece0b..37e917c3d5a6f 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -131,8 +131,9 @@ static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, int32_t depth); +VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, + PyObject *iterable); PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x); - // Misc helpers static inline int check_float_error(PyObject *o) { From 13c0dd7abf4f8d0a4ac72c2d9b4171113da072f2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 22 Nov 2021 21:17:55 +0000 Subject: [PATCH 036/208] Implement vec[i64] equality --- mypyc/lib-rt/vecs/vec_i64.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 1be708a4354f3..6211d7ba8d384 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -40,6 +40,35 @@ static Py_ssize_t vec_length(PyObject *o) { return ((VecI64Object *)o)->len; } +PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { + int cmp = 1; + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (other->ob_type != &VecI64Type) + cmp = 0; + else { + VecI64Object *x = (VecI64Object *)self; + VecI64Object *y = (VecI64Object *)other; + if (x->len != y->len) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < x->len; i++) { + if (x->items[i] != y->items[i]) { + cmp = 0; + break; + } + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, }; @@ -61,6 +90,7 @@ PyTypeObject VecI64Type = { .tp_repr = (reprfunc)vec_i64_repr, .tp_as_sequence = &VecI64Sequence, .tp_as_mapping = &VecI64Mapping, + .tp_richcompare = vec_richcompare, }; static VecI64Object *vec_i64_alloc(Py_ssize_t size) From c8bea0481f4a7b6e6b7c817c7b810349b1a5b217 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 11:40:07 +0000 Subject: [PATCH 037/208] Implement == and != for vec[t] --- mypyc/lib-rt/vecs/vec_i64.c | 4 ++-- mypyc/lib-rt/vecs/vec_t.c | 20 ++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.c | 29 +++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 3 +++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 6211d7ba8d384..6d9a4b2a021c8 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -40,7 +40,7 @@ static Py_ssize_t vec_length(PyObject *o) { return ((VecI64Object *)o)->len; } -PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { +static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { int cmp = 1; PyObject *res; if (op == Py_EQ || op == Py_NE) { @@ -90,7 +90,7 @@ PyTypeObject VecI64Type = { .tp_repr = (reprfunc)vec_i64_repr, .tp_as_sequence = &VecI64Sequence, .tp_as_mapping = &VecI64Mapping, - .tp_richcompare = vec_richcompare, + .tp_richcompare = vec_i64_richcompare, }; static VecI64Object *vec_i64_alloc(Py_ssize_t size) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index e2f342b5cd062..c4abf49a403e3 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -43,6 +43,25 @@ int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } +PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (other->ob_type != &VecTType) { + res = op == Py_EQ ? Py_False : Py_True; + } else { + VecTObject *x = (VecTObject *)self; + VecTObject *y = (VecTObject *)other; + if (x ->item_type != y->item_type) { + res = op == Py_EQ ? Py_False : Py_True; + } else + return vec_generic_richcompare(x->len, x->items, y->len, y->items, op); + } + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -106,6 +125,7 @@ PyTypeObject VecTType = { .tp_repr = (reprfunc)vec_t_repr, .tp_as_sequence = &VecTSequence, .tp_as_mapping = &VecTMapping, + .tp_richcompare = vec_t_richcompare, // TODO: free }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index e163944a05211..688cb716852df 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -226,6 +226,35 @@ PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_ return PyUnicode_Join(sep, l); } +// Generic comparison implementation for vecs with PyObject * items. +PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, + Py_ssize_t other_len, PyObject **other_items, + int op) { + int cmp = 1; + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (len != other_len) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < len; i++) { + int itemcmp = PyObject_RichCompareBool(items[i], other_items[i], Py_EQ); + if (itemcmp < 0) + return NULL; + if (!itemcmp) { + cmp = 0; + break; + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + // Module-level functions static PyObject *vecs_append(PyObject *self, PyObject *args) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 37e917c3d5a6f..792753c58dda9 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -147,5 +147,8 @@ static inline int check_float_error(PyObject *o) { PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals); PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, int verbose); +PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, + Py_ssize_t other_len, PyObject **other_items, + int op); #endif From ff5a6a1a62a00631dbdc9e82d45b75b15f9c0eff Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 12:23:22 +0000 Subject: [PATCH 038/208] More __eq__ support --- mypyc/lib-rt/vecs/vec_t_ext.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 4325b7a1b8493..a7390fa7476b6 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -40,6 +40,27 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } +PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (other->ob_type != &VecTExtType) { + res = op == Py_EQ ? Py_False : Py_True; + } else { + VecTExtObject *x = (VecTExtObject *)self; + VecTExtObject *y = (VecTExtObject *)other; + if (x->item_type != y->item_type + || x->depth != y->depth + || x->optionals != y->optionals) { + res = op == Py_EQ ? Py_False : Py_True; + } else + return vec_generic_richcompare(x->len, x->items, y->len, y->items, op); + } + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + static int VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { @@ -105,6 +126,7 @@ PyTypeObject VecTExtType = { .tp_repr = (reprfunc)vec_t_ext_repr, .tp_as_sequence = &VecTExtSequence, .tp_as_mapping = &VecTExtMapping, + .tp_richcompare = vec_t_ext_richcompare, // TODO: free }; From ce0586dda208093a5f006aef4f2a10834457c559 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 14:31:20 +0000 Subject: [PATCH 039/208] Fix C style --- mypyc/lib-rt/vecs/vec_t.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index c4abf49a403e3..d71e056f43233 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -51,7 +51,7 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { } else { VecTObject *x = (VecTObject *)self; VecTObject *y = (VecTObject *)other; - if (x ->item_type != y->item_type) { + if (x->item_type != y->item_type) { res = op == Py_EQ ? Py_False : Py_True; } else return vec_generic_richcompare(x->len, x->items, y->len, y->items, op); From 4b87612680e76e7a6a0aba3037ed9a0dd588dfe8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 14:31:36 +0000 Subject: [PATCH 040/208] Implement vec[i64] slicing --- mypyc/lib-rt/vecs/vec_i64.c | 179 +++++++++++++++++++++--------------- 1 file changed, 106 insertions(+), 73 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 6d9a4b2a021c8..6461909a338ac 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -2,14 +2,82 @@ #include #include "vecs.h" -static PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw); +static VecI64Object *vec_i64_alloc(Py_ssize_t size) +{ + VecI64Object *v; + /* TODO: Check for overflow */ + v = PyObject_NewVar(VecI64Object, &VecI64Type, size); + if (v == NULL) + return NULL; + return v; +} + +PyObject *Vec_I64_New(Py_ssize_t size) +{ + VecI64Object *v; + v = vec_i64_alloc(size); + if (v == NULL) + return NULL; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = 0; + } + v->len = size; + return (PyObject *)v; +} + +VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { + VecI64Object *v = vec_i64_alloc(0); + if (v == NULL) + return NULL; + v->len = 0; + + PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + Py_DECREF(v); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + int64_t x = PyLong_AsLongLong(item); + Py_DECREF(item); + if (x == -1 && PyErr_Occurred()) { + Py_DECREF(iter); + Py_DECREF(v); + return NULL; + } + v = (VecI64Object *)Vec_I64_Append((PyObject *)v, x); + if (v == NULL) { + Py_DECREF(iter); + Py_DECREF(v); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) { + Py_DECREF(v); + return NULL; + } + return v; +} + +PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { + static char *kwlist[] = {"", NULL}; + PyObject *init = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { + return NULL; + } + if (init == NULL) { + return Vec_I64_New(0); + } else { + return (PyObject *)Vec_I64_FromIterable(init); + } +} PyObject *vec_i64_repr(PyObject *self) { return vec_repr(self, I64TypeObj, 0, 0, 1); } PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { - // TODO: Type check o VecI64Object *v = (VecI64Object *)o; if ((size_t)i < (size_t)v->len) { return PyLong_FromLongLong(v->items[i]); @@ -19,6 +87,41 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { } } + +PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { + VecI64Object *vec = (VecI64Object *)self; + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if ((size_t)i < (size_t)vec->len) { + return PyLong_FromLongLong(vec->items[i]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + } else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) + return NULL; + Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); + VecI64Object *res = vec_i64_alloc(slicelength); + if (res == NULL) + return NULL; + res->len = slicelength; + Py_ssize_t j = start; + for (Py_ssize_t i = 0; i < slicelength; i++) { + res->items[i] = vec->items[j]; + j += step; + } + return (PyObject *)res; + } else { + PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", + item->ob_type->tp_name); + return NULL; + } +} + int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (check_float_error(o)) return -1; @@ -71,6 +174,7 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, + .mp_subscript = vec_i64_subscript, }; static PySequenceMethods VecI64Sequence = { @@ -93,77 +197,6 @@ PyTypeObject VecI64Type = { .tp_richcompare = vec_i64_richcompare, }; -static VecI64Object *vec_i64_alloc(Py_ssize_t size) -{ - VecI64Object *v; - /* TODO: Check for overflow */ - v = PyObject_NewVar(VecI64Object, &VecI64Type, size); - if (v == NULL) - return NULL; - return v; -} - -PyObject *Vec_I64_New(Py_ssize_t size) -{ - VecI64Object *v; - v = vec_i64_alloc(size); - if (v == NULL) - return NULL; - for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = 0; - } - v->len = size; - return (PyObject *)v; -} - -VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { - VecI64Object *v = vec_i64_alloc(0); - if (v == NULL) - return NULL; - v->len = 0; - - PyObject *iter = PyObject_GetIter(iterable); - if (iter == NULL) { - Py_DECREF(v); - return NULL; - } - PyObject *item; - while ((item = PyIter_Next(iter)) != NULL) { - int64_t x = PyLong_AsLongLong(item); - Py_DECREF(item); - if (x == -1 && PyErr_Occurred()) { - Py_DECREF(iter); - Py_DECREF(v); - return NULL; - } - v = (VecI64Object *)Vec_I64_Append((PyObject *)v, x); - if (v == NULL) { - Py_DECREF(iter); - Py_DECREF(v); - return NULL; - } - } - Py_DECREF(iter); - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } - return v; -} - -PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {"", NULL}; - PyObject *init = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { - return NULL; - } - if (init == NULL) { - return Vec_I64_New(0); - } else { - return (PyObject *)Vec_I64_FromIterable(init); - } -} - PyObject *Vec_I64_Append(PyObject *obj, int64_t x) { VecI64Object *vec = (VecI64Object *)obj; Py_ssize_t cap = VEC_SIZE(vec); From ef5b658ba0418c20cf6045aa28f16019b5512d4a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 14:54:22 +0000 Subject: [PATCH 041/208] Small style tweak --- mypyc/lib-rt/vecs/vec_i64.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 6461909a338ac..c451606a0a7aa 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -87,7 +87,6 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { } } - PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { VecI64Object *vec = (VecI64Object *)self; if (PyIndex_Check(item)) { From 3b3df1a1ab89c38d65727616a62f30b5bd733641 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 14:54:34 +0000 Subject: [PATCH 042/208] Implement vec[t] slicing --- mypyc/lib-rt/vecs/vec_t.c | 82 ++++++++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 18 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index d71e056f43233..24edb85f1e04b 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -10,7 +10,30 @@ #include #include "vecs.h" -//static PyObject *vec_t_new(PyTypeObject *self, PyObject *args, PyObject *kw); +static VecTObject *vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { + VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, size); + if (v == NULL) + return NULL; + v->item_type = item_type; + return v; +} + +VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { + VecTObject *v; + v = PyObject_GC_NewVar(VecTObject, &VecTType, size); + //v = VecTType.tp_alloc(&VecTType, size); + if (v == NULL) + return NULL; + + v->item_type = item_type; + v->len = size; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = NULL; + } + + PyObject_GC_Track(v); + return v; +} PyObject *vec_t_repr(PyObject *self) { VecTObject *v = (VecTObject *)self; @@ -29,6 +52,45 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } +PyObject *vec_t_subscript(PyObject *self, PyObject *item) { + VecTObject *vec = (VecTObject *)self; + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if ((size_t)i < (size_t)vec->len) { + PyObject *item = vec->items[i]; + Py_INCREF(item); + return item; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + } else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) + return NULL; + Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); + VecTObject *res = vec_t_alloc(slicelength, vec->item_type); + if (res == NULL) + return NULL; + res->len = slicelength; + Py_ssize_t j = start; + for (Py_ssize_t i = 0; i < slicelength; i++) { + PyObject *item = vec->items[j]; + Py_INCREF(item); + res->items[i] = item; + j += step; + } + PyObject_GC_Track(res); + return (PyObject *)res; + } else { + PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", + item->ob_type->tp_name); + return NULL; + } +} + int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTObject *v = (VecTObject *)self; if (!VecT_ItemCheck(v, o)) @@ -104,6 +166,7 @@ static Py_ssize_t vec_length(PyObject *o) { static PyMappingMethods VecTMapping = { .mp_length = vec_length, + .mp_subscript = vec_t_subscript, }; static PySequenceMethods VecTSequence = { @@ -129,23 +192,6 @@ PyTypeObject VecTType = { // TODO: free }; -VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { - VecTObject *v; - v = PyObject_GC_NewVar(VecTObject, &VecTType, size); - //v = VecTType.tp_alloc(&VecTType, size); - if (v == NULL) - return NULL; - - v->item_type = item_type; - v->len = size; - for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = NULL; - } - - PyObject_GC_Track(v); - return v; -} - VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, 0); if (v == NULL) From b76d9a138059a402721637b3f040212d0fe5061b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 15:26:55 +0000 Subject: [PATCH 043/208] Implement remaining vec slicing --- mypyc/lib-rt/vecs/vec_t.c | 2 + mypyc/lib-rt/vecs/vec_t_ext.c | 94 +++++++++++++++++++++++++++-------- 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 24edb85f1e04b..d437e419bc14c 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -10,6 +10,8 @@ #include #include "vecs.h" +// Alloc a partially initialized vec. Caller *must* initialize len and items, and +// call PyObject_GC_Track(). static VecTObject *vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, size); if (v == NULL) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index a7390fa7476b6..144c243595524 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,6 +9,39 @@ #include #include "vecs.h" +// Alloc a partially initialized vec. Caller *must* initialize len and items, and +// call PyObject_GC_Track(). +static VecTExtObject *vec_t_ext_alloc(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, + int32_t depth) { + VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); + if (v == NULL) + return NULL; + v->item_type = item_type; + v->optionals = optionals; + v->depth = depth; + return v; +} + +VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, + int32_t depth) { + VecTExtObject *v; + v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); + //v = VecTType.tp_alloc(&VecTType, size); + if (v == NULL) + return NULL; + + v->item_type = item_type; + v->len = size; + for (Py_ssize_t i = 0; i < size; i++) { + v->items[i] = NULL; + } + v->optionals = optionals; + v->depth = depth; + + PyObject_GC_Track(v); + return v; +} + PyObject *vec_t_ext_repr(PyObject *self) { VecTExtObject *v = (VecTExtObject *)self; return vec_repr(self, v->item_type, v->depth, v->optionals, 1); @@ -26,6 +59,46 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { } } +PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { + VecTExtObject *vec = (VecTExtObject *)self; + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if ((size_t)i < (size_t)vec->len) { + PyObject *item = vec->items[i]; + Py_INCREF(item); + return item; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + } else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) + return NULL; + Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); + VecTExtObject *res = vec_t_ext_alloc(slicelength, vec->item_type, vec->optionals, + vec->depth); + if (res == NULL) + return NULL; + res->len = slicelength; + Py_ssize_t j = start; + for (Py_ssize_t i = 0; i < slicelength; i++) { + PyObject *item = vec->items[j]; + Py_INCREF(item); + res->items[i] = item; + j += step; + } + PyObject_GC_Track(res); + return (PyObject *)res; + } else { + PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", + item->ob_type->tp_name); + return NULL; + } +} + int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTExtObject *v = (VecTExtObject *)self; if (!VecTExt_ItemCheck(v, o)) @@ -105,6 +178,7 @@ static Py_ssize_t vec_ext_length(PyObject *o) { static PyMappingMethods VecTExtMapping = { .mp_length = vec_ext_length, + .mp_subscript = vec_t_ext_subscript, }; static PySequenceMethods VecTExtSequence = { @@ -130,26 +204,6 @@ PyTypeObject VecTExtType = { // TODO: free }; -VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, - int32_t depth) { - VecTExtObject *v; - v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); - //v = VecTType.tp_alloc(&VecTType, size); - if (v == NULL) - return NULL; - - v->item_type = item_type; - v->len = size; - for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = NULL; - } - v->optionals = optionals; - v->depth = depth; - - PyObject_GC_Track(v); - return v; -} - VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, PyObject *iterable) { VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, 0); From a755479b9b2ec92a6c998cce5891e7b34cef8996 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 15:55:43 +0000 Subject: [PATCH 044/208] Implement vec[i64].remove(x) --- mypyc/lib-rt/vecs/vec_i64.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index c451606a0a7aa..dfb0b277d2965 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -171,6 +171,27 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } +static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { + if (check_float_error(arg)) + return NULL; + VecI64Object *v = (VecI64Object *)self; + long long x = PyLong_AsLongLong(arg); + if (x == -1 && PyErr_Occurred()) + return NULL; + + for (Py_ssize_t i = 0; i < v->len; i++) { + if (v->items[i] == x) { + for (; i < v->len - 1; i++) { + v->items[i] = v->items[i + 1]; + } + v->len--; + Py_RETURN_NONE; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + return NULL; +} + static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, .mp_subscript = vec_i64_subscript, @@ -181,6 +202,11 @@ static PySequenceMethods VecI64Sequence = { .sq_ass_item = vec_i64_ass_item, }; +static PyMethodDef vec_i64_methods[] = { + {"remove", vec_i64_remove, METH_O, NULL}, + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + PyTypeObject VecI64Type = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec[i64]", @@ -194,6 +220,7 @@ PyTypeObject VecI64Type = { .tp_as_sequence = &VecI64Sequence, .tp_as_mapping = &VecI64Mapping, .tp_richcompare = vec_i64_richcompare, + .tp_methods = vec_i64_methods, }; PyObject *Vec_I64_Append(PyObject *obj, int64_t x) { From 11e33158fffed461a55913071e08303588522f26 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 16:28:14 +0000 Subject: [PATCH 045/208] Implement vec[t].remove(x) --- mypyc/lib-rt/vecs/vec_t.c | 13 +++++++++++++ mypyc/lib-rt/vecs/vecs.c | 24 ++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 1 + 3 files changed, 38 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index d437e419bc14c..180abc06ee1f3 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -126,6 +126,13 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } +static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { + VecTObject *v = (VecTObject *)self; + if (!VecT_ItemCheck(v, arg)) + return NULL; + return vec_generic_remove(&v->len, v->items, arg); +} + static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -176,6 +183,11 @@ static PySequenceMethods VecTSequence = { .sq_ass_item = vec_t_ass_item, }; +static PyMethodDef vec_t_methods[] = { + {"remove", vec_t_remove, METH_O, NULL}, + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + PyTypeObject VecTType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", @@ -191,6 +203,7 @@ PyTypeObject VecTType = { .tp_as_sequence = &VecTSequence, .tp_as_mapping = &VecTMapping, .tp_richcompare = vec_t_richcompare, + .tp_methods = vec_t_methods, // TODO: free }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 688cb716852df..a6b26a18a3f2e 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -255,6 +255,30 @@ PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, return res; } +PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) { + for (Py_ssize_t i = 0; i < *len; i++) { + int match = 0; + if (items[i] == item) + match = 1; + else { + int itemcmp = PyObject_RichCompareBool(items[i], item, Py_EQ); + if (itemcmp < 0) + return NULL; + match = itemcmp; + } + if (match) { + Py_DECREF(items[i]); + for (; i < *len - 1; i++) { + items[i] = items[i + 1]; + } + (*len)--; + Py_RETURN_NONE; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + return NULL; +} + // Module-level functions static PyObject *vecs_append(PyObject *self, PyObject *args) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 792753c58dda9..a08495fc8cc8f 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -150,5 +150,6 @@ PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_ PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, Py_ssize_t other_len, PyObject **other_items, int op); +PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); #endif From ad3bb8f69d2b065fd53216b2b1de00179d414665 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 16:52:59 +0000 Subject: [PATCH 046/208] Fix possible issue where vec being compared changes size --- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_t_ext.c | 2 +- mypyc/lib-rt/vecs/vecs.c | 10 +++++----- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 180abc06ee1f3..d3a2af16245b5 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -118,7 +118,7 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { if (x->item_type != y->item_type) { res = op == Py_EQ ? Py_False : Py_True; } else - return vec_generic_richcompare(x->len, x->items, y->len, y->items, op); + return vec_generic_richcompare(&x->len, x->items, &y->len, y->items, op); } } else res = Py_NotImplemented; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 144c243595524..2eadc409b85c0 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -126,7 +126,7 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { || x->optionals != y->optionals) { res = op == Py_EQ ? Py_False : Py_True; } else - return vec_generic_richcompare(x->len, x->items, y->len, y->items, op); + return vec_generic_richcompare(&x->len, x->items, &y->len, y->items, op); } } else res = Py_NotImplemented; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index a6b26a18a3f2e..41aef1b133a3c 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -227,16 +227,16 @@ PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_ } // Generic comparison implementation for vecs with PyObject * items. -PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, - Py_ssize_t other_len, PyObject **other_items, +PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, + Py_ssize_t *other_len, PyObject **other_items, int op) { int cmp = 1; PyObject *res; if (op == Py_EQ || op == Py_NE) { - if (len != other_len) { + if (*len != *other_len) { cmp = 0; } else { - for (Py_ssize_t i = 0; i < len; i++) { + for (Py_ssize_t i = 0; i < *len && i < *other_len; i++) { int itemcmp = PyObject_RichCompareBool(items[i], other_items[i], Py_EQ); if (itemcmp < 0) return NULL; @@ -267,7 +267,7 @@ PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) match = itemcmp; } if (match) { - Py_DECREF(items[i]); + Py_CLEAR(items[i]); for (; i < *len - 1; i++) { items[i] = items[i + 1]; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index a08495fc8cc8f..b0f49e390d338 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -147,8 +147,8 @@ static inline int check_float_error(PyObject *o) { PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals); PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, int verbose); -PyObject *vec_generic_richcompare(Py_ssize_t len, PyObject **items, - Py_ssize_t other_len, PyObject **other_items, +PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, + Py_ssize_t *other_len, PyObject **other_items, int op); PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); From a7102823d95cd101899e77aa248093286965b23c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 16:54:29 +0000 Subject: [PATCH 047/208] Fix refcounting and gc issues The reference counting bug could result in segfaults. --- mypyc/lib-rt/vecs/vec_t.c | 4 ++++ mypyc/lib-rt/vecs/vec_t_ext.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index d3a2af16245b5..1f99a2ff85499 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -16,6 +16,7 @@ static VecTObject *vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, size); if (v == NULL) return NULL; + Py_INCREF(item_type); v->item_type = item_type; return v; } @@ -27,6 +28,7 @@ VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { if (v == NULL) return NULL; + Py_INCREF(item_type); v->item_type = item_type; v->len = size; for (Py_ssize_t i = 0; i < size; i++) { @@ -212,7 +214,9 @@ VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { if (v == NULL) return NULL; v->len = 0; + Py_INCREF(item_type); v->item_type = item_type; + PyObject_GC_Track(v); PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 2eadc409b85c0..ec835a9cb2ed5 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -16,6 +16,7 @@ static VecTExtObject *vec_t_ext_alloc(Py_ssize_t size, PyTypeObject *item_type, VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); if (v == NULL) return NULL; + Py_INCREF(item_type); v->item_type = item_type; v->optionals = optionals; v->depth = depth; @@ -30,6 +31,7 @@ VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t o if (v == NULL) return NULL; + Py_INCREF(item_type); v->item_type = item_type; v->len = size; for (Py_ssize_t i = 0; i < size; i++) { @@ -210,9 +212,11 @@ VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals if (v == NULL) return NULL; v->len = 0; + Py_INCREF(item_type); v->item_type = item_type; v->optionals = optionals; v->depth = depth; + PyObject_GC_Track(v); PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { From 4e59657960f56c46bc97ff83703c4ba9ef0f1d19 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 16:55:30 +0000 Subject: [PATCH 048/208] Finish up implementation of vec.remove --- mypyc/lib-rt/vecs/vec_t_ext.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index ec835a9cb2ed5..9a4617611a86a 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -136,6 +136,13 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { return res; } +static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { + VecTExtObject *v = (VecTExtObject *)self; + if (!VecTExt_ItemCheck(v, arg)) + return NULL; + return vec_generic_remove(&v->len, v->items, arg); +} + static int VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { @@ -188,6 +195,11 @@ static PySequenceMethods VecTExtSequence = { .sq_ass_item = vec_t_ext_ass_item, }; +static PyMethodDef vec_t_ext_methods[] = { + {"remove", vec_t_ext_remove, METH_O, NULL}, + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + PyTypeObject VecTExtType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", @@ -203,6 +215,7 @@ PyTypeObject VecTExtType = { .tp_as_sequence = &VecTExtSequence, .tp_as_mapping = &VecTExtMapping, .tp_richcompare = vec_t_ext_richcompare, + .tp_methods = vec_t_ext_methods, // TODO: free }; From 639617f0e672bd22ffe2b9344561a00baad19efd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Nov 2021 16:55:52 +0000 Subject: [PATCH 049/208] Clean up dealloc code a bit --- mypyc/lib-rt/vecs/vec_t.c | 10 ++++------ mypyc/lib-rt/vecs/vec_t_ext.c | 12 ++++-------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 1f99a2ff85499..30d5c8b59f053 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -160,12 +160,10 @@ VecT_dealloc(VecTObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecT_dealloc) - VecT_clear(self); - //Py_DECREF(self->item_type); - //for (Py_ssize_t i = 0; i < self->len; i++) { - // Py_XDECREF(self->items[i]); - //} - //PyObject_GC_Del(self); + Py_DECREF(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + Py_XDECREF(self->items[i]); + } Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END } diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 9a4617611a86a..a186bd852f767 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -148,7 +148,6 @@ VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { Py_VISIT(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { - // TODO: NULL checks? Py_VISIT(self->items[i]); } return 0; @@ -159,7 +158,6 @@ VecTExt_clear(VecTExtObject *self) { Py_CLEAR(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { - // TODO: NULL checks? Py_CLEAR(self->items[i]); } return 0; @@ -170,12 +168,10 @@ VecTExt_dealloc(VecTExtObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) - VecTExt_clear(self); - //Py_DECREF(self->item_type); - //for (Py_ssize_t i = 0; i < self->len; i++) { - // Py_XDECREF(self->items[i]); - //} - //PyObject_GC_Del(self); + Py_DECREF(self->item_type); + for (Py_ssize_t i = 0; i < self->len; i++) { + Py_XDECREF(self->items[i]); + } Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END } From e7246a0fb08b62cbb5015d3db0f3b1526165e4ad Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 09:01:47 +0000 Subject: [PATCH 050/208] Implement vec[i64].pop --- mypyc/lib-rt/vecs/vec_i64.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index dfb0b277d2965..f0d51e243d26e 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -192,6 +192,29 @@ static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { return NULL; } +static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:vec", &index)) + return NULL; + + VecI64Object *v = (VecI64Object *)self; + if (index < 0) + index += v->len; + + if (index < 0 || index >= v->len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + + int64_t item = v->items[index]; + for (Py_ssize_t i = index; i < v->len - 1; i++) + v->items[i] = v->items[i + 1]; + + v->len--; + + return PyLong_FromLongLong(item); +} + static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, .mp_subscript = vec_i64_subscript, @@ -204,6 +227,7 @@ static PySequenceMethods VecI64Sequence = { static PyMethodDef vec_i64_methods[] = { {"remove", vec_i64_remove, METH_O, NULL}, + {"pop", vec_i64_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; From d9a270a93850f45f1cb3721f6076c37c69fe32cb Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 09:16:59 +0000 Subject: [PATCH 051/208] Implement vec[t].pop --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 6 ++++++ mypyc/lib-rt/vecs/vecs.c | 22 ++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 1 + 4 files changed, 30 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index f0d51e243d26e..ec3ff069d5871 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -194,7 +194,7 @@ static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:vec", &index)) + if (!PyArg_ParseTuple(args, "|n:pop", &index)) return NULL; VecI64Object *v = (VecI64Object *)self; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 30d5c8b59f053..ddecc6f046d81 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -135,6 +135,11 @@ static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { return vec_generic_remove(&v->len, v->items, arg); } +static PyObject *vec_t_pop(PyObject *self, PyObject *args) { + VecTObject *v = (VecTObject *)self; + return vec_generic_pop(&v->len, v->items, args); +} + static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -185,6 +190,7 @@ static PySequenceMethods VecTSequence = { static PyMethodDef vec_t_methods[] = { {"remove", vec_t_remove, METH_O, NULL}, + {"pop", vec_t_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 41aef1b133a3c..5bac00ac504b5 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -279,6 +279,28 @@ PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) return NULL; } +PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, PyObject *args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; + + if (index < 0) + index += *len; + + if (index < 0 || index >= *len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + + PyObject *item = items[index]; + for (Py_ssize_t i = index; i < *len - 1; i++) + items[i] = items[i + 1]; + + (*len)--; + + return item; +} + // Module-level functions static PyObject *vecs_append(PyObject *self, PyObject *args) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index b0f49e390d338..dc46eb91eed44 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -151,5 +151,6 @@ PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); +PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, PyObject *args); #endif From 4a3ecfdf83dd0cf5b6977ef7b230b5e10f8fff20 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 09:19:18 +0000 Subject: [PATCH 052/208] Finish implementation of pop --- mypyc/lib-rt/vecs/vec_t_ext.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index a186bd852f767..6a83dafaccd3f 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -143,6 +143,11 @@ static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { return vec_generic_remove(&v->len, v->items, arg); } +static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { + VecTExtObject *v = (VecTExtObject *)self; + return vec_generic_pop(&v->len, v->items, args); +} + static int VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { @@ -193,6 +198,7 @@ static PySequenceMethods VecTExtSequence = { static PyMethodDef vec_t_ext_methods[] = { {"remove", vec_t_ext_remove, METH_O, NULL}, + {"pop", vec_t_ext_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; From 9bd0028b5943e872ee3196d121480da7dfa4b894 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 14:53:17 +0000 Subject: [PATCH 053/208] Add VecObject struct --- mypyc/lib-rt/vecs/vecs.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index dc46eb91eed44..6ff2732a537d8 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -4,6 +4,12 @@ #define PY_SSIZE_T_CLEAN #include +// Arbitrary vec object (only shared bits) +typedef struct _VecObject { + PyObject_VAR_HEAD + Py_ssize_t len; +} VecObject; + // vec[i64] typedef struct _VecI64Object { PyObject_VAR_HEAD From 78fbf952712e617e1b0f7bb158869d948ea57554 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 15:01:31 +0000 Subject: [PATCH 054/208] Include vec[t] in capsule --- mypyc/lib-rt/vecs/vec_t.c | 14 ++++++++++---- mypyc/lib-rt/vecs/vecs.c | 5 +++-- mypyc/lib-rt/vecs/vecs.h | 7 ++++--- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index ddecc6f046d81..0f4cf98d025c8 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -21,7 +21,7 @@ static VecTObject *vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { return v; } -VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { +PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) { VecTObject *v; v = PyObject_GC_NewVar(VecTObject, &VecTType, size); //v = VecTType.tp_alloc(&VecTType, size); @@ -29,14 +29,14 @@ VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type) { return NULL; Py_INCREF(item_type); - v->item_type = item_type; + v->item_type = (PyTypeObject *)item_type; v->len = size; for (Py_ssize_t i = 0; i < size; i++) { v->items[i] = NULL; } PyObject_GC_Track(v); - return v; + return (PyObject *)v; } PyObject *vec_t_repr(PyObject *self) { @@ -263,7 +263,7 @@ PyObject *Vec_T_Append(PyObject *obj, PyObject *x) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTObject *new = Vec_T_New(new_size, vec->item_type); + VecTObject *new = (VecTObject *)Vec_T_New(new_size, (PyObject *)vec->item_type); if (new == NULL) return NULL; memcpy(new->items, vec->items, sizeof(PyObject *) * len); @@ -274,3 +274,9 @@ PyObject *Vec_T_Append(PyObject *obj, PyObject *x) { return (PyObject *)new; } } + +VecTFeatures TFeatures = { + &VecTType, + Vec_T_New, + Vec_T_Append, +}; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 5bac00ac504b5..61a96bd141f47 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -27,7 +27,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) VecProxy *p = (VecProxy *)self; if (p->optionals == 0 && p->depth == 0) { if (init == NULL) { - return (PyObject *)Vec_T_New(0, (PyTypeObject *)p->item_type); + return (PyObject *)Vec_T_New(0, (PyObject *)p->item_type); } else { return (PyObject *)Vec_T_FromIterable((PyTypeObject *)p->item_type, init); } @@ -352,7 +352,8 @@ static PyModuleDef vecsmodule = { }; static VecCapsule Capsule = { - &I64Features + &I64Features, + &TFeatures, }; PyMODINIT_FUNC diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 6ff2732a537d8..af578b2402d2d 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -54,7 +54,7 @@ typedef struct _VecI64Features { // T can be a class type, T | None or vec[T] (or a combination of these) typedef struct _VecTFeatures { PyTypeObject *type; - PyObject *(*alloc)(Py_ssize_t); + PyObject *(*alloc)(Py_ssize_t, PyObject *item_type); PyObject *(*append)(PyObject *, PyObject *); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); @@ -67,7 +67,7 @@ typedef struct _VecTFeatures { typedef struct { VecI64Features *i64; - // VecTFeatures *t; + VecTFeatures *t; // VecTExtFeatures *t_ext; } VecCapsule; @@ -80,6 +80,7 @@ extern PyTypeObject VecTType; extern PyTypeObject VecTExtType; extern VecI64Features I64Features; extern PyTypeObject *I64TypeObj; +extern VecTFeatures TFeatures; // vec[i64] operations @@ -105,7 +106,7 @@ static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { } } -VecTObject *Vec_T_New(Py_ssize_t size, PyTypeObject *item_type); +PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable); PyObject *Vec_T_Append(PyObject *obj, PyObject *x); From 745613ae6e8a96be928d78fd1d5291d958643338 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 28 Nov 2021 16:46:02 +0000 Subject: [PATCH 055/208] Include vec[t] ext in capsule --- mypyc/lib-rt/vecs/vec_t_ext.c | 16 +++++++++++----- mypyc/lib-rt/vecs/vecs.c | 3 ++- mypyc/lib-rt/vecs/vecs.h | 26 +++++++++++++++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 6a83dafaccd3f..8972f96e3359b 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -23,8 +23,7 @@ static VecTExtObject *vec_t_ext_alloc(Py_ssize_t size, PyTypeObject *item_type, return v; } -VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, - int32_t depth) { +PyObject *Vec_T_Ext_New(Py_ssize_t size, PyObject *item_type, int32_t optionals, int32_t depth) { VecTExtObject *v; v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); //v = VecTType.tp_alloc(&VecTType, size); @@ -32,7 +31,7 @@ VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t o return NULL; Py_INCREF(item_type); - v->item_type = item_type; + v->item_type = (PyTypeObject *)item_type; v->len = size; for (Py_ssize_t i = 0; i < size; i++) { v->items[i] = NULL; @@ -41,7 +40,7 @@ VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t o v->depth = depth; PyObject_GC_Track(v); - return v; + return (PyObject *)v; } PyObject *vec_t_ext_repr(PyObject *self) { @@ -274,7 +273,8 @@ PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTExtObject *new = Vec_T_Ext_New(new_size, vec->item_type, vec->optionals, vec->depth); + VecTExtObject *new = (VecTExtObject *)Vec_T_Ext_New( + new_size, (PyObject *)vec->item_type, vec->optionals, vec->depth); if (new == NULL) return NULL; memcpy(new->items, vec->items, sizeof(PyObject *) * len); @@ -285,3 +285,9 @@ PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { return (PyObject *)new; } } + +VecTExtFeatures TExtFeatures = { + &VecTExtType, + Vec_T_Ext_New, + Vec_T_Ext_Append, +}; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 61a96bd141f47..f2b30085fd07b 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -33,7 +33,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } } else { if (init == NULL) { - return (PyObject *)Vec_T_Ext_New(0, (PyTypeObject *)p->item_type, p->optionals, + return (PyObject *)Vec_T_Ext_New(0, (PyObject *)p->item_type, p->optionals, p->depth); } else { return (PyObject *)Vec_T_Ext_FromIterable((PyTypeObject *)p->item_type, p->optionals, @@ -354,6 +354,7 @@ static PyModuleDef vecsmodule = { static VecCapsule Capsule = { &I64Features, &TFeatures, + &TExtFeatures, }; PyMODINIT_FUNC diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index af578b2402d2d..b475bac40ee50 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -51,10 +51,10 @@ typedef struct _VecI64Features { // vec[T] operations + type object (stored in a capsule) // -// T can be a class type, T | None or vec[T] (or a combination of these) +// T is a class type typedef struct _VecTFeatures { PyTypeObject *type; - PyObject *(*alloc)(Py_ssize_t, PyObject *item_type); + PyObject *(*alloc)(Py_ssize_t, PyObject *); PyObject *(*append)(PyObject *, PyObject *); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); @@ -65,10 +65,26 @@ typedef struct _VecTFeatures { // iter? } VecTFeatures; +// vec[T] operations for complex item types + type object (stored in a capsule) +// +// T can be T | None or vec[T] (or a combination of these) +typedef struct _VecTExtFeatures { + PyTypeObject *type; + PyObject *(*alloc)(Py_ssize_t, PyObject *, int optional, int depth); + PyObject *(*append)(PyObject *, PyObject *); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*slice)(PyObject *, int64_t, int64_t); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, PyObject *); + // PyObject *(*pop)(PyObject *); + // bool (*remove)(PyObject *, PyObject *); + // iter? +} VecTExtFeatures; + typedef struct { VecI64Features *i64; VecTFeatures *t; - // VecTExtFeatures *t_ext; + VecTExtFeatures *t_ext; } VecCapsule; #define VEC_SIZE(v) ((v)->ob_base.ob_size) @@ -81,6 +97,7 @@ extern PyTypeObject VecTExtType; extern VecI64Features I64Features; extern PyTypeObject *I64TypeObj; extern VecTFeatures TFeatures; +extern VecTExtFeatures TExtFeatures; // vec[i64] operations @@ -136,8 +153,7 @@ static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { } } -VecTExtObject *Vec_T_Ext_New(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, - int32_t depth); +PyObject *Vec_T_Ext_New(Py_ssize_t size, PyObject *item_type, int32_t optionals, int32_t depth); VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, PyObject *iterable); PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x); From f9e55ebb9eba27c7cdf623ddd7e162b71eacb6d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 4 Dec 2021 14:20:13 +0000 Subject: [PATCH 056/208] Minor tweak --- mypyc/lib-rt/vecs/vecs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index b475bac40ee50..b7a76f8469e44 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -70,7 +70,7 @@ typedef struct _VecTFeatures { // T can be T | None or vec[T] (or a combination of these) typedef struct _VecTExtFeatures { PyTypeObject *type; - PyObject *(*alloc)(Py_ssize_t, PyObject *, int optional, int depth); + PyObject *(*alloc)(Py_ssize_t, PyObject *, int optionals, int depth); PyObject *(*append)(PyObject *, PyObject *); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); From f9b731a1d304dbf2a297b33c4069718a46c3b9b4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Dec 2021 13:35:50 +0000 Subject: [PATCH 057/208] Add pop(index) to external API --- mypyc/lib-rt/vecs/vec_i64.c | 22 +++++++++++++++------- mypyc/lib-rt/vecs/vec_t.c | 10 ++++++++-- mypyc/lib-rt/vecs/vec_t_ext.c | 10 ++++++++-- mypyc/lib-rt/vecs/vecs.c | 15 +++++++++------ mypyc/lib-rt/vecs/vecs.h | 13 +++++++++---- 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index ec3ff069d5871..c0a16b550b052 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -192,18 +192,14 @@ static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { return NULL; } -static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { - Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:pop", &index)) - return NULL; - +static int64_t Vec_I64_Pop(PyObject *self, Py_ssize_t index) { VecI64Object *v = (VecI64Object *)self; if (index < 0) index += v->len; if (index < 0 || index >= v->len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; + return MYPYC_INT_ERROR; } int64_t item = v->items[index]; @@ -211,8 +207,19 @@ static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { v->items[i] = v->items[i + 1]; v->len--; + return item; +} + +static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; + + int64_t res = Vec_I64_Pop(self, index); + if (res == MYPYC_INT_ERROR && PyErr_Occurred()) + return NULL; - return PyLong_FromLongLong(item); + return PyLong_FromLongLong(res); } static PyMappingMethods VecI64Mapping = { @@ -272,4 +279,5 @@ VecI64Features I64Features = { &VecI64Type, Vec_I64_New, Vec_I64_Append, + Vec_I64_Pop, }; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 0f4cf98d025c8..7eb61e3686f48 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -135,9 +135,14 @@ static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { return vec_generic_remove(&v->len, v->items, arg); } +static PyObject *Vec_T_Pop(PyObject *self, Py_ssize_t index) { + VecTObject *v = (VecTObject *)self; + return vec_generic_pop(&v->len, v->items, index); +} + static PyObject *vec_t_pop(PyObject *self, PyObject *args) { VecTObject *v = (VecTObject *)self; - return vec_generic_pop(&v->len, v->items, args); + return vec_generic_pop_wrapper(&v->len, v->items, args); } static int @@ -165,7 +170,7 @@ VecT_dealloc(VecTObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecT_dealloc) - Py_DECREF(self->item_type); + Py_XDECREF(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { Py_XDECREF(self->items[i]); } @@ -279,4 +284,5 @@ VecTFeatures TFeatures = { &VecTType, Vec_T_New, Vec_T_Append, + Vec_T_Pop, }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 8972f96e3359b..baa57960eb056 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -142,9 +142,14 @@ static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { return vec_generic_remove(&v->len, v->items, arg); } +static PyObject *Vec_T_Ext_Pop(PyObject *self, Py_ssize_t index) { + VecTExtObject *v = (VecTExtObject *)self; + return vec_generic_pop(&v->len, v->items, index); +} + static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { VecTExtObject *v = (VecTExtObject *)self; - return vec_generic_pop(&v->len, v->items, args); + return vec_generic_pop_wrapper(&v->len, v->items, args); } static int @@ -172,7 +177,7 @@ VecTExt_dealloc(VecTExtObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) - Py_DECREF(self->item_type); + Py_XDECREF(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { Py_XDECREF(self->items[i]); } @@ -290,4 +295,5 @@ VecTExtFeatures TExtFeatures = { &VecTExtType, Vec_T_Ext_New, Vec_T_Ext_Append, + Vec_T_Ext_Pop, }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index f2b30085fd07b..5c22ee12a0903 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -279,11 +279,7 @@ PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) return NULL; } -PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, PyObject *args) { - Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:pop", &index)) - return NULL; - +PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { if (index < 0) index += *len; @@ -297,10 +293,17 @@ PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, PyObject *args) { items[i] = items[i + 1]; (*len)--; - return item; } +PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *args) { + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; + + return vec_generic_pop(len, items, index); +} + // Module-level functions static PyObject *vecs_append(PyObject *self, PyObject *args) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index b7a76f8469e44..868b1f7ef8ab3 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -4,6 +4,10 @@ #define PY_SSIZE_T_CLEAN #include +// Magic (native) integer return value on exception. Caller must also +// use PyErr_Occurred() since this overlaps with valid integer values. +#define MYPYC_INT_ERROR -113 + // Arbitrary vec object (only shared bits) typedef struct _VecObject { PyObject_VAR_HEAD @@ -40,11 +44,11 @@ typedef struct _VecI64Features { PyTypeObject *type; PyObject *(*alloc)(Py_ssize_t); PyObject *(*append)(PyObject *, int64_t); + int64_t (*pop)(PyObject *, Py_ssize_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); - // int64_t (*pop)(PyObject *); // bool (*remove)(PyObject *, int64_t); // iter? } VecI64Features; @@ -56,11 +60,11 @@ typedef struct _VecTFeatures { PyTypeObject *type; PyObject *(*alloc)(Py_ssize_t, PyObject *); PyObject *(*append)(PyObject *, PyObject *); + PyObject *(*pop)(PyObject *, Py_ssize_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); - // PyObject *(*pop)(PyObject *); // bool (*remove)(PyObject *, PyObject *); // iter? } VecTFeatures; @@ -72,11 +76,11 @@ typedef struct _VecTExtFeatures { PyTypeObject *type; PyObject *(*alloc)(Py_ssize_t, PyObject *, int optionals, int depth); PyObject *(*append)(PyObject *, PyObject *); + PyObject *(*pop)(PyObject *, Py_ssize_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); - // PyObject *(*pop)(PyObject *); // bool (*remove)(PyObject *, PyObject *); // iter? } VecTExtFeatures; @@ -174,6 +178,7 @@ PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); -PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, PyObject *args); +PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *args); +PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); #endif From 61036db7084c9261ce229692b0528aa480a06368 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Dec 2021 17:49:32 +0000 Subject: [PATCH 058/208] Add vec[i64] remove to API --- mypyc/lib-rt/vecs/vec_i64.c | 24 ++++++++++++++++-------- mypyc/lib-rt/vecs/vecs.h | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index c0a16b550b052..9e23090d12e22 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -171,13 +171,8 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { - if (check_float_error(arg)) - return NULL; +static int Vec_I64_Remove(PyObject *self, int64_t x) { VecI64Object *v = (VecI64Object *)self; - long long x = PyLong_AsLongLong(arg); - if (x == -1 && PyErr_Occurred()) - return NULL; for (Py_ssize_t i = 0; i < v->len; i++) { if (v->items[i] == x) { @@ -185,11 +180,23 @@ static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { v->items[i] = v->items[i + 1]; } v->len--; - Py_RETURN_NONE; + return 1; } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return NULL; + return 0; +} + +static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { + if (check_float_error(arg)) + return NULL; + long long x = PyLong_AsLongLong(arg); + if (x == -1 && PyErr_Occurred()) + return NULL; + + if (!Vec_I64_Remove(self, x)) + return NULL; + Py_RETURN_NONE; } static int64_t Vec_I64_Pop(PyObject *self, Py_ssize_t index) { @@ -280,4 +287,5 @@ VecI64Features I64Features = { Vec_I64_New, Vec_I64_Append, Vec_I64_Pop, + Vec_I64_Remove, }; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 868b1f7ef8ab3..f2463770e97d7 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -45,11 +45,11 @@ typedef struct _VecI64Features { PyObject *(*alloc)(Py_ssize_t); PyObject *(*append)(PyObject *, int64_t); int64_t (*pop)(PyObject *, Py_ssize_t); + int (*remove)(PyObject *, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); - // bool (*remove)(PyObject *, int64_t); // iter? } VecI64Features; From 67dc4ec516a4dc9f8a84689513fe2d5a97754bbc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 11 Dec 2021 17:54:24 +0000 Subject: [PATCH 059/208] Add remaining remove variants to external API --- mypyc/lib-rt/vecs/vec_t.c | 10 +++++++++- mypyc/lib-rt/vecs/vec_t_ext.c | 10 +++++++++- mypyc/lib-rt/vecs/vecs.c | 8 ++++---- mypyc/lib-rt/vecs/vecs.h | 6 +++--- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 7eb61e3686f48..baf1f394d81c0 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -128,11 +128,18 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } +static int Vec_T_Remove(PyObject *self, PyObject *arg) { + VecTObject *v = (VecTObject *)self; + return vec_generic_remove(&v->len, v->items, arg); +} + static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { VecTObject *v = (VecTObject *)self; if (!VecT_ItemCheck(v, arg)) return NULL; - return vec_generic_remove(&v->len, v->items, arg); + if (!vec_generic_remove(&v->len, v->items, arg)) + return NULL; + Py_RETURN_NONE; } static PyObject *Vec_T_Pop(PyObject *self, Py_ssize_t index) { @@ -285,4 +292,5 @@ VecTFeatures TFeatures = { Vec_T_New, Vec_T_Append, Vec_T_Pop, + Vec_T_Remove, }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index baa57960eb056..4ea42bb5bf82f 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -135,11 +135,18 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { return res; } +static int Vec_T_Ext_Remove(PyObject *self, PyObject *arg) { + VecTExtObject *v = (VecTExtObject *)self; + return vec_generic_remove(&v->len, v->items, arg); +} + static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { VecTExtObject *v = (VecTExtObject *)self; if (!VecTExt_ItemCheck(v, arg)) return NULL; - return vec_generic_remove(&v->len, v->items, arg); + if (!vec_generic_remove(&v->len, v->items, arg)) + return NULL; + Py_RETURN_NONE; } static PyObject *Vec_T_Ext_Pop(PyObject *self, Py_ssize_t index) { @@ -296,4 +303,5 @@ VecTExtFeatures TExtFeatures = { Vec_T_Ext_New, Vec_T_Ext_Append, Vec_T_Ext_Pop, + Vec_T_Ext_Remove, }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 5c22ee12a0903..dc8d313fbdf48 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -255,7 +255,7 @@ PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, return res; } -PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) { +int vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) { for (Py_ssize_t i = 0; i < *len; i++) { int match = 0; if (items[i] == item) @@ -263,7 +263,7 @@ PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) else { int itemcmp = PyObject_RichCompareBool(items[i], item, Py_EQ); if (itemcmp < 0) - return NULL; + return 0; match = itemcmp; } if (match) { @@ -272,11 +272,11 @@ PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) items[i] = items[i + 1]; } (*len)--; - Py_RETURN_NONE; + return 1; } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return NULL; + return 0; } PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index f2463770e97d7..c773b9306543e 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -61,11 +61,11 @@ typedef struct _VecTFeatures { PyObject *(*alloc)(Py_ssize_t, PyObject *); PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); + int (*remove)(PyObject *, PyObject *); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); - // bool (*remove)(PyObject *, PyObject *); // iter? } VecTFeatures; @@ -77,11 +77,11 @@ typedef struct _VecTExtFeatures { PyObject *(*alloc)(Py_ssize_t, PyObject *, int optionals, int depth); PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); + int (*remove)(PyObject *, PyObject *); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); - // bool (*remove)(PyObject *, PyObject *); // iter? } VecTExtFeatures; @@ -177,7 +177,7 @@ PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_ PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); -PyObject *vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); +int vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *args); PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); From 7b30bc8928e5a231485227d47cf6259c6124e1ab Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 28 Dec 2021 11:21:17 +0000 Subject: [PATCH 060/208] Add API function for vec slicing --- mypyc/lib-rt/vecs/vec_i64.c | 23 +++++++++++++++++++++++ mypyc/lib-rt/vecs/vec_t.c | 26 ++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vec_t_ext.c | 26 ++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 9 ++++++--- 4 files changed, 81 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 9e23090d12e22..13584de6068bb 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -87,6 +87,28 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { } } +PyObject *Vec_I64_Slice(PyObject *self, int64_t start, int64_t end) { + VecI64Object *vec = (VecI64Object *)self; + if (start < 0) + start += vec->len; + if (end < 0) + end += vec->len; + if (end < start) + end = start; + if (start < 0) + start = 0; + if (end > vec->len) + end = vec->len; + int64_t slicelength = end - start; + VecI64Object *res = vec_i64_alloc(slicelength); + if (res == NULL) + return NULL; + res->len = slicelength; + for (Py_ssize_t i = 0; i < slicelength; i++) + res->items[i] = vec->items[start + i]; + return (PyObject *)res; +} + PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { VecI64Object *vec = (VecI64Object *)self; if (PyIndex_Check(item)) { @@ -288,4 +310,5 @@ VecI64Features I64Features = { Vec_I64_Append, Vec_I64_Pop, Vec_I64_Remove, + Vec_I64_Slice, }; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index baf1f394d81c0..5af4d4c2920a6 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -56,6 +56,31 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } +PyObject *Vec_T_Slice(PyObject *self, int64_t start, int64_t end) { + VecTObject *vec = (VecTObject *)self; + if (start < 0) + start += vec->len; + if (end < 0) + end += vec->len; + if (end < start) + end = start; + if (start < 0) + start = 0; + if (end > vec->len) + end = vec->len; + int64_t slicelength = end - start; + VecTObject *res = vec_t_alloc(slicelength, vec->item_type); + if (res == NULL) + return NULL; + res->len = slicelength; + for (Py_ssize_t i = 0; i < slicelength; i++) { + PyObject *item = vec->items[start + i]; + Py_INCREF(item); + res->items[i] = item; + } + return (PyObject *)res; +} + PyObject *vec_t_subscript(PyObject *self, PyObject *item) { VecTObject *vec = (VecTObject *)self; if (PyIndex_Check(item)) { @@ -293,4 +318,5 @@ VecTFeatures TFeatures = { Vec_T_Append, Vec_T_Pop, Vec_T_Remove, + Vec_T_Slice, }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 4ea42bb5bf82f..5ce5c74c49dd2 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -60,6 +60,31 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { } } +PyObject *Vec_T_Ext_Slice(PyObject *self, int64_t start, int64_t end) { + VecTExtObject *vec = (VecTExtObject *)self; + if (start < 0) + start += vec->len; + if (end < 0) + end += vec->len; + if (end < start) + end = start; + if (start < 0) + start = 0; + if (end > vec->len) + end = vec->len; + int64_t slicelength = end - start; + VecTExtObject *res = vec_t_ext_alloc(slicelength, vec->item_type, vec->optionals, vec->depth); + if (res == NULL) + return NULL; + res->len = slicelength; + for (Py_ssize_t i = 0; i < slicelength; i++) { + PyObject *item = vec->items[start + i]; + Py_INCREF(item); + res->items[i] = item; + } + return (PyObject *)res; +} + PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { VecTExtObject *vec = (VecTExtObject *)self; if (PyIndex_Check(item)) { @@ -304,4 +329,5 @@ VecTExtFeatures TExtFeatures = { Vec_T_Ext_Append, Vec_T_Ext_Pop, Vec_T_Ext_Remove, + Vec_T_Ext_Slice, }; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index c773b9306543e..ccf3b897b9511 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -46,8 +46,9 @@ typedef struct _VecI64Features { PyObject *(*append)(PyObject *, int64_t); int64_t (*pop)(PyObject *, Py_ssize_t); int (*remove)(PyObject *, int64_t); + // TODO: Py_ssize_t + PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); - // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); // iter? @@ -62,8 +63,9 @@ typedef struct _VecTFeatures { PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); int (*remove)(PyObject *, PyObject *); + // TODO: Py_ssize_t + PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); - // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); // iter? @@ -78,8 +80,9 @@ typedef struct _VecTExtFeatures { PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); int (*remove)(PyObject *, PyObject *); + // TODO: Py_ssize_t + PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); - // PyObject *(*slice)(PyObject *, int64_t, int64_t); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); // iter? From 9264c89073e8fa0808969f22904d7fd8e215dbe8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Apr 2023 16:06:57 +0100 Subject: [PATCH 061/208] [WIP] Move to new vec representation for vec[i64] Other item types are currently unsupported. --- mypyc/lib-rt/vecs/vec_i64.c | 248 +++++++++++++++++++++--------------- mypyc/lib-rt/vecs/vecs.c | 21 ++- mypyc/lib-rt/vecs/vecs.h | 115 +++++++++++++---- 3 files changed, 253 insertions(+), 131 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 13584de6068bb..1cc1509d55bce 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -2,38 +2,51 @@ #include #include "vecs.h" -static VecI64Object *vec_i64_alloc(Py_ssize_t size) +static VecI64 vec_i64_alloc(Py_ssize_t size) { - VecI64Object *v; + VecbufI64Object *buf; /* TODO: Check for overflow */ - v = PyObject_NewVar(VecI64Object, &VecI64Type, size); - if (v == NULL) + buf = PyObject_NewVar(VecbufI64Object, &VecbufI64Type, size); + if (buf == NULL) + return Vec_I64_Error(); + VecI64 res = { .buf = buf }; + return res; +} + +static void vec_i64_dealloc(VecI64Object *self) { + Py_CLEAR(self->vec.buf); + PyObject_Del(self); +} + +PyObject *Vec_I64_Box(VecI64 vec) { + VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); + if (obj == NULL) return NULL; - return v; + obj->vec = vec; + return (PyObject *)obj; } -PyObject *Vec_I64_New(Py_ssize_t size) +VecI64 Vec_I64_New(Py_ssize_t size) { - VecI64Object *v; - v = vec_i64_alloc(size); - if (v == NULL) - return NULL; + VecI64 vec = vec_i64_alloc(size); + if (VEC_IS_ERROR(vec)) + return vec; for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = 0; + vec.buf->items[i] = 0; } - v->len = size; - return (PyObject *)v; + vec.len = size; + return vec; } -VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { - VecI64Object *v = vec_i64_alloc(0); - if (v == NULL) +PyObject *Vec_I64_FromIterable(PyObject *iterable) { + VecI64 v = vec_i64_alloc(0); + if (VEC_IS_ERROR(v)) return NULL; - v->len = 0; + v.len = 0; PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } PyObject *item; @@ -42,22 +55,22 @@ VecI64Object *Vec_I64_FromIterable(PyObject *iterable) { Py_DECREF(item); if (x == -1 && PyErr_Occurred()) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); return NULL; } - v = (VecI64Object *)Vec_I64_Append((PyObject *)v, x); - if (v == NULL) { + v = Vec_I64_Append(v, x); + if (VEC_IS_ERROR(v)) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); return NULL; } } Py_DECREF(iter); if (PyErr_Occurred()) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } - return v; + return Vec_I64_Box(v); } PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { @@ -67,7 +80,7 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return NULL; } if (init == NULL) { - return Vec_I64_New(0); + return Vec_I64_Box(Vec_I64_New(0)); } else { return (PyObject *)Vec_I64_FromIterable(init); } @@ -78,45 +91,44 @@ PyObject *vec_i64_repr(PyObject *self) { } PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { - VecI64Object *v = (VecI64Object *)o; - if ((size_t)i < (size_t)v->len) { - return PyLong_FromLongLong(v->items[i]); + VecI64 v = ((VecI64Object *)o)->vec; + if ((size_t)i < (size_t)v.len) { + return PyLong_FromLongLong(v.buf->items[i]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; } } -PyObject *Vec_I64_Slice(PyObject *self, int64_t start, int64_t end) { - VecI64Object *vec = (VecI64Object *)self; +VecI64 Vec_I64_Slice(VecI64 vec, int64_t start, int64_t end) { if (start < 0) - start += vec->len; + start += vec.len; if (end < 0) - end += vec->len; + end += vec.len; if (end < start) end = start; if (start < 0) start = 0; - if (end > vec->len) - end = vec->len; + if (end > vec.len) + end = vec.len; int64_t slicelength = end - start; - VecI64Object *res = vec_i64_alloc(slicelength); - if (res == NULL) - return NULL; - res->len = slicelength; + VecI64 res = vec_i64_alloc(slicelength); + if (VEC_IS_ERROR(res)) + return Vec_I64_Error(); + res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) - res->items[i] = vec->items[start + i]; - return (PyObject *)res; + res.buf->items[i] = vec.buf->items[start + i]; + return res; } PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { - VecI64Object *vec = (VecI64Object *)self; + VecI64 vec = ((VecI64Object *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; - if ((size_t)i < (size_t)vec->len) { - return PyLong_FromLongLong(vec->items[i]); + if ((size_t)i < (size_t)vec.len) { + return PyLong_FromLongLong(vec.buf->items[i]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -125,17 +137,17 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); - VecI64Object *res = vec_i64_alloc(slicelength); - if (res == NULL) + Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); + VecI64 res = vec_i64_alloc(slicelength); + if (VEC_IS_ERROR(res)) return NULL; - res->len = slicelength; + res.len = slicelength; Py_ssize_t j = start; for (Py_ssize_t i = 0; i < slicelength; i++) { - res->items[i] = vec->items[j]; + res.buf->items[i] = vec.buf->items[j]; j += step; } - return (PyObject *)res; + return Vec_I64_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -146,12 +158,12 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (check_float_error(o)) return -1; - VecI64Object *v = (VecI64Object *)self; - if ((size_t)i < (size_t)v->len) { + VecI64 v = ((VecI64Object *)self)->vec; + if ((size_t)i < (size_t)v.len) { long long x = PyLong_AsLongLong(o); if (x == -1 && PyErr_Occurred()) return -1; - v->items[i] = x; + v.buf->items[i] = x; return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); @@ -161,7 +173,7 @@ int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { static Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o - return ((VecI64Object *)o)->len; + return ((VecI64Object *)o)->vec.len; } static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { @@ -171,13 +183,13 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { if (other->ob_type != &VecI64Type) cmp = 0; else { - VecI64Object *x = (VecI64Object *)self; - VecI64Object *y = (VecI64Object *)other; - if (x->len != y->len) { + VecI64 x = ((VecI64Object *)self)->vec; + VecI64 y = ((VecI64Object *)other)->vec; + if (x.len != y.len) { cmp = 0; } else { - for (Py_ssize_t i = 0; i < x->len; i++) { - if (x->items[i] != y->items[i]) { + for (Py_ssize_t i = 0; i < x.len; i++) { + if (x.buf->items[i] != y.buf->items[i]) { cmp = 0; break; } @@ -193,20 +205,19 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static int Vec_I64_Remove(PyObject *self, int64_t x) { - VecI64Object *v = (VecI64Object *)self; - - for (Py_ssize_t i = 0; i < v->len; i++) { - if (v->items[i] == x) { - for (; i < v->len - 1; i++) { - v->items[i] = v->items[i + 1]; +static VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { + for (Py_ssize_t i = 0; i < v.len; i++) { + if (v.buf->items[i] == x) { + for (; i < v.len - 1; i++) { + v.buf->items[i] = v.buf->items[i + 1]; } - v->len--; - return 1; + v.len--; + VEC_INCREF(v); + return v; } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return 0; + return Vec_I64_Error(); } static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { @@ -216,27 +227,29 @@ static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { if (x == -1 && PyErr_Occurred()) return NULL; - if (!Vec_I64_Remove(self, x)) + VecI64Object *vec = (VecI64Object *)self; + VecI64 res = Vec_I64_Remove(vec->vec, x); + if (VEC_IS_ERROR(res)) return NULL; - Py_RETURN_NONE; + return Vec_I64_Box(res); } -static int64_t Vec_I64_Pop(PyObject *self, Py_ssize_t index) { - VecI64Object *v = (VecI64Object *)self; +static VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { if (index < 0) - index += v->len; + index += v.len; - if (index < 0 || index >= v->len) { + if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - return MYPYC_INT_ERROR; + return Vec_I64_Error(); } - int64_t item = v->items[index]; - for (Py_ssize_t i = index; i < v->len - 1; i++) - v->items[i] = v->items[i + 1]; + *result = v.buf->items[index]; + for (Py_ssize_t i = index; i < v.len - 1; i++) + v.buf->items[i] = v.buf->items[i + 1]; - v->len--; - return item; + v.len--; + VEC_INCREF(v); + return v; } static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { @@ -244,11 +257,31 @@ static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { if (!PyArg_ParseTuple(args, "|n:pop", &index)) return NULL; - int64_t res = Vec_I64_Pop(self, index); - if (res == MYPYC_INT_ERROR && PyErr_Occurred()) + int64_t x; + VecI64Object *vec = (VecI64Object *)self; + VecI64 vec2 = Vec_I64_Pop(vec->vec, index, &x); + if (VEC_IS_ERROR(vec2)) + return NULL; + + PyObject *item0 = Vec_I64_Box(vec2); + if (item0 == NULL) return NULL; + PyObject *item1 = PyLong_FromLongLong(x); + if (item1 == NULL) { + Py_DECREF(item0); + return NULL; + } + + PyObject *res = PyTuple_New(2); + if (res == NULL) { + Py_DECREF(item0); + Py_DECREF(item1); + return NULL; + } - return PyLong_FromLongLong(res); + PyTuple_SetItem(res, 0, item0); + PyTuple_SetItem(res, 1, item1); + return res; } static PyMappingMethods VecI64Mapping = { @@ -267,15 +300,27 @@ static PyMethodDef vec_i64_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecI64Type = { +PyTypeObject VecbufI64Type = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec[i64]", + .tp_name = "vecbuf[i64]", .tp_doc = "vec doc", .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), .tp_itemsize = sizeof(int64_t), .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = vec_i64_new, + //.tp_new = vecbuf_i64_new, //?? .tp_free = PyObject_Del, +}; + +PyTypeObject VecI64Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec[i64]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecI64Object), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vec_i64_new, + //.tp_free = PyObject_Del, + .tp_dealloc = (destructor)vec_i64_dealloc, .tp_repr = (reprfunc)vec_i64_repr, .tp_as_sequence = &VecI64Sequence, .tp_as_mapping = &VecI64Mapping, @@ -283,29 +328,28 @@ PyTypeObject VecI64Type = { .tp_methods = vec_i64_methods, }; -PyObject *Vec_I64_Append(PyObject *obj, int64_t x) { - VecI64Object *vec = (VecI64Object *)obj; - Py_ssize_t cap = VEC_SIZE(vec); - Py_ssize_t len = vec->len; - if (len < cap) { - vec->items[len] = x; - vec->len = len + 1; - return (PyObject *)vec; +VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { + Py_ssize_t cap = VEC_CAP(vec); + if (vec.len < cap) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; } else { Py_ssize_t new_size = 2 * cap + 1; - VecI64Object *new = vec_i64_alloc(new_size); - if (new == NULL) - return NULL; - memcpy(new->items, vec->items, sizeof(int64_t) * len); - new->items[len] = x; - new->len = len + 1; - Py_DECREF(vec); - return (PyObject *)new; + VecI64 new = vec_i64_alloc(new_size); + if (VEC_IS_ERROR(new)) + return Vec_I64_Error(); + new.len = vec.len + 1; + memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); + new.buf->items[vec.len] = x; + Py_DECREF(vec.buf); + return new; } } VecI64Features I64Features = { &VecI64Type, + &VecbufI64Type, Vec_I64_New, Vec_I64_Append, Vec_I64_Pop, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index dc8d313fbdf48..16142eab0fb31 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -24,6 +24,8 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { return NULL; } + return NULL; +#if 0 VecProxy *p = (VecProxy *)self; if (p->optionals == 0 && p->depth == 0) { if (init == NULL) { @@ -40,6 +42,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) p->depth, init); } } +#endif } static int @@ -321,8 +324,13 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) if (x == -1 && PyErr_Occurred()) { return NULL; } - Py_INCREF(vec); - return Vec_I64_Append(vec, x); + VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); + v = Vec_I64_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; + return Vec_I64_Box(v); +#if 0 } else if (VecT_Check(vec)) { VecTObject *v = (VecTObject *)vec; if (!VecT_ItemCheck(v, item)) @@ -335,6 +343,7 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return NULL; Py_INCREF(vec); return Vec_T_Ext_Append(vec, item); +#endif } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; @@ -356,8 +365,8 @@ static PyModuleDef vecsmodule = { static VecCapsule Capsule = { &I64Features, - &TFeatures, - &TExtFeatures, + NULL, // TODO: &TFeatures, + NULL, // TODO: &TExtFeatures, }; PyMODINIT_FUNC @@ -378,12 +387,16 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecProxyType) < 0) return NULL; +#if 0 if (PyType_Ready(&VecTType) < 0) return NULL; if (PyType_Ready(&VecTExtType) < 0) return NULL; +#endif if (PyType_Ready(&VecI64Type) < 0) return NULL; + if (PyType_Ready(&VecbufI64Type) < 0) + return NULL; PyObject *m = PyModule_Create(&vecsmodule); if (m == NULL) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index ccf3b897b9511..8dc76bbd0dae4 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -8,46 +8,90 @@ // use PyErr_Occurred() since this overlaps with valid integer values. #define MYPYC_INT_ERROR -113 -// Arbitrary vec object (only shared bits) -typedef struct _VecObject { - PyObject_VAR_HEAD - Py_ssize_t len; -} VecObject; -// vec[i64] -typedef struct _VecI64Object { +// Buffer objects + + +// vecbuf[i64] +typedef struct _VecbufI64Object { PyObject_VAR_HEAD - Py_ssize_t len; int64_t items[1]; -} VecI64Object; +} VecbufI64Object; -// Simple generic vec: vec[t] when t is a type object -typedef struct _VecTObject { +// Simple generic vecbuf: vecbuf[t] when t is a type object +typedef struct _VecbufTObject { PyObject_VAR_HEAD - Py_ssize_t len; PyTypeObject *item_type; PyObject *items[1]; -} VecTObject; +} VecbufTObject; // Extended generic vec type: vec[t | None], vec[vec[...]], etc. -typedef struct _VecTExtObject { +typedef struct _VecbufTExtObject { PyObject_VAR_HEAD - Py_ssize_t len; PyTypeObject *item_type; int32_t depth; // Number of nested VecTExt or VecT types int32_t optionals; // Flags for optional types on each nesting level PyObject *items[1]; +} VecbufTExtObject; + + +// Unboxed vec objects + + +typedef struct _VecI64 { + Py_ssize_t len; + VecbufI64Object *buf; +} VecI64; + +typedef struct _VecT { + Py_ssize_t len; + VecbufTObject *buf; +} VecT; + +typedef struct _VecTExt { + Py_ssize_t len; + VecbufTExtObject *buf; +} VecTExt; + + +// Boxed vec objects + + +// Arbitrary boxed vec object (only shared bits) +typedef struct _VecObject { + PyObject_HEAD + Py_ssize_t len; +} VecObject; + +// Boxed vec[i64] +typedef struct _VecI64Object { + PyObject_HEAD + VecI64 vec; +} VecI64Object; + +// Simple boxed generic vecbuf: vecbuf[t] when t is a type object +typedef struct _VecTObject { + PyObject_HEAD + VecT vec; +} VecTObject; + +// Extended generic vec type: vec[t | None], vec[vec[...]], etc. +typedef struct _VecTExtObject { + PyObject_HEAD + VecTExt vec; } VecTExtObject; -// vec[i64] operations + type object (stored in a capsule) + +// vec[i64] operations + type objects (stored in a capsule) typedef struct _VecI64Features { - PyTypeObject *type; - PyObject *(*alloc)(Py_ssize_t); - PyObject *(*append)(PyObject *, int64_t); - int64_t (*pop)(PyObject *, Py_ssize_t); - int (*remove)(PyObject *, int64_t); + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI64 (*alloc)(Py_ssize_t); + VecI64 (*append)(VecI64, int64_t); + VecI64 (*pop)(VecI64, Py_ssize_t, int64_t *result); + VecI64 (*remove)(VecI64, int64_t); // TODO: Py_ssize_t - PyObject *(*slice)(PyObject *, int64_t, int64_t); + VecI64 (*slice)(VecI64, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); @@ -58,7 +102,8 @@ typedef struct _VecI64Features { // // T is a class type typedef struct _VecTFeatures { - PyTypeObject *type; + PyTypeObject *boxed_type; + PyTypeObject *buf_type; PyObject *(*alloc)(Py_ssize_t, PyObject *); PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); @@ -75,7 +120,8 @@ typedef struct _VecTFeatures { // // T can be T | None or vec[T] (or a combination of these) typedef struct _VecTExtFeatures { - PyTypeObject *type; + PyTypeObject *boxed_type; + PyTypeObject *buf_type; PyObject *(*alloc)(Py_ssize_t, PyObject *, int optionals, int depth); PyObject *(*append)(PyObject *, PyObject *); PyObject *(*pop)(PyObject *, Py_ssize_t); @@ -95,12 +141,26 @@ typedef struct { } VecCapsule; #define VEC_SIZE(v) ((v)->ob_base.ob_size) +#define VEC_CAP(v) ((v).buf->ob_base.ob_size) +#define VEC_IS_ERROR(v) ((v).len < 0) +#define VEC_DECREF(v) Py_DECREF((v).buf) +#define VEC_INCREF(v) Py_INCREF((v).buf) + +inline VecI64 Vec_I64_Error() { + VecI64 v = { .len = -1 }; + return v; +} // Type objects +extern PyTypeObject VecbufI64Type; +extern PyTypeObject VecbufTType; +extern PyTypeObject VecbufTExtType; + extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; extern PyTypeObject VecTExtType; + extern VecI64Features I64Features; extern PyTypeObject *I64TypeObj; extern VecTFeatures TFeatures; @@ -112,7 +172,8 @@ static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } -PyObject *Vec_I64_Append(PyObject *obj, int64_t x); +PyObject *Vec_I64_Box(VecI64); +VecI64 Vec_I64_Append(VecI64, int64_t x); // vec[t] operations (simple) @@ -120,6 +181,8 @@ static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; } +#if 0 + static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { if (PyObject_TypeCheck(item, v->item_type)) return 1; @@ -160,6 +223,8 @@ static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { } } +#endif + PyObject *Vec_T_Ext_New(Py_ssize_t size, PyObject *item_type, int32_t optionals, int32_t depth); VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, PyObject *iterable); From 86768bd2afdc2b5a6d00b7f93d15074a2f77ca04 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Apr 2023 16:08:50 +0100 Subject: [PATCH 062/208] Include boxing op in the capsule (i64 only for now) --- mypyc/lib-rt/vecs/vec_i64.c | 1 + mypyc/lib-rt/vecs/vecs.h | 1 + 2 files changed, 2 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 1cc1509d55bce..5d25cab13b4ee 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -351,6 +351,7 @@ VecI64Features I64Features = { &VecI64Type, &VecbufI64Type, Vec_I64_New, + Vec_I64_Box, Vec_I64_Append, Vec_I64_Pop, Vec_I64_Remove, diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 8dc76bbd0dae4..3cb05447b9635 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -87,6 +87,7 @@ typedef struct _VecI64Features { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecI64 (*alloc)(Py_ssize_t); + PyObject *(*box)(VecI64); VecI64 (*append)(VecI64, int64_t); VecI64 (*pop)(VecI64, Py_ssize_t, int64_t *result); VecI64 (*remove)(VecI64, int64_t); From dd11f84f6f12c5a64bca06c7ba75622d6268f7d9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 10 Apr 2023 16:15:37 +0100 Subject: [PATCH 063/208] Use NULL for empty buffer --- mypyc/lib-rt/vecs/vec_i64.c | 19 ++++++++++++------- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 5d25cab13b4ee..f1d768395e390 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -6,9 +6,13 @@ static VecI64 vec_i64_alloc(Py_ssize_t size) { VecbufI64Object *buf; /* TODO: Check for overflow */ - buf = PyObject_NewVar(VecbufI64Object, &VecbufI64Type, size); - if (buf == NULL) - return Vec_I64_Error(); + if (size == 0) { + buf = NULL; + } else { + buf = PyObject_NewVar(VecbufI64Object, &VecbufI64Type, size); + if (buf == NULL) + return Vec_I64_Error(); + } VecI64 res = { .buf = buf }; return res; } @@ -329,20 +333,21 @@ PyTypeObject VecI64Type = { }; VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { - Py_ssize_t cap = VEC_CAP(vec); - if (vec.len < cap) { + if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; return vec; } else { + Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; Py_ssize_t new_size = 2 * cap + 1; VecI64 new = vec_i64_alloc(new_size); if (VEC_IS_ERROR(new)) return Vec_I64_Error(); new.len = vec.len + 1; - memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); + if (vec.len > 0) + memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); new.buf->items[vec.len] = x; - Py_DECREF(vec.buf); + Py_XDECREF(vec.buf); return new; } } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 3cb05447b9635..35ffb4f7c52d4 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -144,8 +144,8 @@ typedef struct { #define VEC_SIZE(v) ((v)->ob_base.ob_size) #define VEC_CAP(v) ((v).buf->ob_base.ob_size) #define VEC_IS_ERROR(v) ((v).len < 0) -#define VEC_DECREF(v) Py_DECREF((v).buf) -#define VEC_INCREF(v) Py_INCREF((v).buf) +#define VEC_DECREF(v) Py_XDECREF((v).buf) +#define VEC_INCREF(v) Py_XINCREF((v).buf) inline VecI64 Vec_I64_Error() { VecI64 v = { .len = -1 }; From 3f6d05632ade2693f5f7f50a003f85cd764397c5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 14:10:36 +0100 Subject: [PATCH 064/208] Partial implementation of vec[T] for type objects Implementations of remove() and pop() are still missing. --- mypyc/lib-rt/vecs/vec_i64.c | 6 +- mypyc/lib-rt/vecs/vec_t.c | 258 ++++++++++++++++++++++-------------- mypyc/lib-rt/vecs/vecs.c | 34 +++-- mypyc/lib-rt/vecs/vecs.h | 33 +++-- 4 files changed, 201 insertions(+), 130 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index f1d768395e390..9c60f9b3260f2 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -2,6 +2,7 @@ #include #include "vecs.h" +// Alloc a partially initialized vec. Caller *must* initialize len. static VecI64 vec_i64_alloc(Py_ssize_t size) { VecbufI64Object *buf; @@ -30,8 +31,7 @@ PyObject *Vec_I64_Box(VecI64 vec) { return (PyObject *)obj; } -VecI64 Vec_I64_New(Py_ssize_t size) -{ +VecI64 Vec_I64_New(Py_ssize_t size) { VecI64 vec = vec_i64_alloc(size); if (VEC_IS_ERROR(vec)) return vec; @@ -118,7 +118,7 @@ VecI64 Vec_I64_Slice(VecI64 vec, int64_t start, int64_t end) { int64_t slicelength = end - start; VecI64 res = vec_i64_alloc(slicelength); if (VEC_IS_ERROR(res)) - return Vec_I64_Error(); + return res; res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) res.buf->items[i] = vec.buf->items[start + i]; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 5af4d4c2920a6..ad71c5d276758 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -10,44 +10,49 @@ #include #include "vecs.h" -// Alloc a partially initialized vec. Caller *must* initialize len and items, and -// call PyObject_GC_Track(). -static VecTObject *vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { - VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, size); - if (v == NULL) - return NULL; +// Alloc a partially initialized vec. Caller *must* initialize len and buf->items. +static VecT vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { + VecbufTObject *buf = PyObject_GC_NewVar(VecbufTObject, &VecbufTType, size); + if (buf == NULL) + return Vec_T_Error(); Py_INCREF(item_type); - v->item_type = item_type; - return v; + buf->item_type = item_type; + VecT res = { .buf = buf }; + PyObject_GC_Track(buf); + return res; } -PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type) { - VecTObject *v; - v = PyObject_GC_NewVar(VecTObject, &VecTType, size); - //v = VecTType.tp_alloc(&VecTType, size); - if (v == NULL) +// TODO: vec_t_dealloc + +PyObject *Vec_T_Box(VecT vec) { + VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); + if (obj == NULL) return NULL; + obj->vec = vec; + PyObject_GC_Track(obj); + return (PyObject *)obj; +} - Py_INCREF(item_type); - v->item_type = (PyTypeObject *)item_type; - v->len = size; +VecT Vec_T_New(Py_ssize_t size, PyObject *item_type) { + VecT vec = vec_t_alloc(size, (PyTypeObject *)item_type); + if (VEC_IS_ERROR(vec)) + return vec; for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = NULL; + vec.buf->items[i] = NULL; } - - PyObject_GC_Track(v); - return (PyObject *)v; + vec.len = size; + return vec; } PyObject *vec_t_repr(PyObject *self) { VecTObject *v = (VecTObject *)self; - return vec_repr(self, v->item_type, 0, 0, 1); + return vec_repr(self, v->vec.buf->item_type, 0, 0, 1); } PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { - VecTObject *v = (VecTObject *)o; - if ((size_t)i < (size_t)v->len) { - PyObject *item = v->items[i]; + VecT v = ((VecTObject *)o)->vec; + if ((size_t)i < (size_t)v.len) { + PyObject *item = v.buf->items[i]; Py_INCREF(item); return item; } else { @@ -56,39 +61,38 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } -PyObject *Vec_T_Slice(PyObject *self, int64_t start, int64_t end) { - VecTObject *vec = (VecTObject *)self; +VecT Vec_T_Slice(VecT vec, int64_t start, int64_t end) { if (start < 0) - start += vec->len; + start += vec.len; if (end < 0) - end += vec->len; + end += vec.len; if (end < start) end = start; if (start < 0) start = 0; - if (end > vec->len) - end = vec->len; + if (end > vec.len) + end = vec.len; int64_t slicelength = end - start; - VecTObject *res = vec_t_alloc(slicelength, vec->item_type); - if (res == NULL) - return NULL; - res->len = slicelength; + VecT res = vec_t_alloc(slicelength, vec.buf->item_type); + if (VEC_IS_ERROR(res)) + return res; + res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec->items[start + i]; + PyObject *item = vec.buf->items[start + i]; Py_INCREF(item); - res->items[i] = item; + res.buf->items[i] = item; } - return (PyObject *)res; + return res; } PyObject *vec_t_subscript(PyObject *self, PyObject *item) { - VecTObject *vec = (VecTObject *)self; + VecT vec = ((VecTObject *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; - if ((size_t)i < (size_t)vec->len) { - PyObject *item = vec->items[i]; + if ((size_t)i < (size_t)vec.len) { + PyObject *item = vec.buf->items[i]; Py_INCREF(item); return item; } else { @@ -99,20 +103,19 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); - VecTObject *res = vec_t_alloc(slicelength, vec->item_type); - if (res == NULL) + Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); + VecT res = vec_t_alloc(slicelength, vec.buf->item_type); + if (VEC_IS_ERROR(res)) return NULL; - res->len = slicelength; + res.len = slicelength; Py_ssize_t j = start; for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec->items[j]; + PyObject *item = vec.buf->items[j]; Py_INCREF(item); - res->items[i] = item; + res.buf->items[i] = item; j += step; } - PyObject_GC_Track(res); - return (PyObject *)res; + return Vec_T_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -121,12 +124,12 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { } int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - VecTObject *v = (VecTObject *)self; + VecT v = ((VecTObject *)self)->vec; if (!VecT_ItemCheck(v, o)) return -1; - if ((size_t)i < (size_t)v->len) { + if ((size_t)i < (size_t)v.len) { Py_INCREF(o); - v->items[i] = o; + v.buf->items[i] = o; return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); @@ -140,12 +143,14 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { if (other->ob_type != &VecTType) { res = op == Py_EQ ? Py_False : Py_True; } else { - VecTObject *x = (VecTObject *)self; - VecTObject *y = (VecTObject *)other; - if (x->item_type != y->item_type) { + VecT x = ((VecTObject *)self)->vec; + VecT y = ((VecTObject *)other)->vec; + if (x.buf->item_type != y.buf->item_type) { res = op == Py_EQ ? Py_False : Py_True; - } else - return vec_generic_richcompare(&x->len, x->items, &y->len, y->items, op); + } else { + // TODO: why pointers to len? + return vec_generic_richcompare(&x.len, x.buf->items, &y.len, y.buf->items, op); + } } } else res = Py_NotImplemented; @@ -153,35 +158,47 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static int Vec_T_Remove(PyObject *self, PyObject *arg) { - VecTObject *v = (VecTObject *)self; - return vec_generic_remove(&v->len, v->items, arg); +static VecT Vec_T_Remove(VecT v, PyObject *arg) { + return Vec_T_Error(); // TODO implement + //VecTObject *v = (VecTObject *)self; + //return vec_generic_remove(&v->len, v->items, arg); } static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { + Py_RETURN_NONE; // TODO implement +#if 0 VecTObject *v = (VecTObject *)self; if (!VecT_ItemCheck(v, arg)) return NULL; if (!vec_generic_remove(&v->len, v->items, arg)) return NULL; - Py_RETURN_NONE; +#endif } -static PyObject *Vec_T_Pop(PyObject *self, Py_ssize_t index) { - VecTObject *v = (VecTObject *)self; - return vec_generic_pop(&v->len, v->items, index); +static VecT Vec_T_Pop(VecT self, Py_ssize_t index, PyObject **result) { + return Vec_T_Error(); // TODO implement + //VecTObject *v = (VecTObject *)self; + //return vec_generic_pop(&v->len, v->items, index); } static PyObject *vec_t_pop(PyObject *self, PyObject *args) { - VecTObject *v = (VecTObject *)self; - return vec_generic_pop_wrapper(&v->len, v->items, args); + return NULL; // TODO implement + //VecTObject *v = (VecTObject *)self; + //return vec_generic_pop_wrapper(&v->len, v->items, args); } static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->vec.buf); + return 0; +} + +static int +VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) { Py_VISIT(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_VISIT(self->items[i]); } return 0; @@ -189,9 +206,16 @@ VecT_traverse(VecTObject *self, visitproc visit, void *arg) static int VecT_clear(VecTObject *self) +{ + Py_CLEAR(self->vec.buf); + return 0; +} + +static int +VecbufT_clear(VecbufTObject *self) { Py_CLEAR(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_CLEAR(self->items[i]); } return 0; @@ -202,9 +226,19 @@ VecT_dealloc(VecTObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecT_dealloc) - Py_XDECREF(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { - Py_XDECREF(self->items[i]); + Py_CLEAR(self->vec.buf); + Py_TYPE(self)->tp_free((PyObject *)self); + Py_TRASHCAN_END +} + +static void +VecbufT_dealloc(VecbufTObject *self) +{ + PyObject_GC_UnTrack(self); + Py_TRASHCAN_BEGIN(self, VecbufT_dealloc) + Py_CLEAR(self->item_type); + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_CLEAR(self->items[i]); } Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END @@ -212,7 +246,7 @@ VecT_dealloc(VecTObject *self) static Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o - return ((VecTObject *)o)->len; + return ((VecTObject *)o)->vec.len; } static PyMappingMethods VecTMapping = { @@ -231,12 +265,26 @@ static PyMethodDef vec_t_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; +PyTypeObject VecbufTType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecbuf", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecbufTObject) - sizeof(PyObject *), + .tp_itemsize = sizeof(PyObject *), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = (traverseproc)VecbufT_traverse, + //.tp_new = vecbuf_i64_new, //?? + .tp_free = PyObject_GC_Del, + .tp_clear = (inquiry)VecbufT_clear, + .tp_dealloc = (destructor)VecbufT_dealloc, +}; + PyTypeObject VecTType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecTObject) - sizeof(PyObject *), - .tp_itemsize = sizeof(PyObject *), + .tp_basicsize = sizeof(VecTObject), + .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_traverse = (traverseproc)VecT_traverse, .tp_clear = (inquiry)VecT_clear, @@ -250,71 +298,75 @@ PyTypeObject VecTType = { // TODO: free }; -VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { - VecTObject *v = PyObject_GC_NewVar(VecTObject, &VecTType, 0); - if (v == NULL) +PyObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { + VecT v = vec_t_alloc(0, item_type); + if (VEC_IS_ERROR(v)) return NULL; - v->len = 0; + v.len = 0; Py_INCREF(item_type); - v->item_type = item_type; - PyObject_GC_Track(v); + v.buf->item_type = item_type; PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { if (!VecT_ItemCheck(v, item)) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = (VecTObject *)Vec_T_Append((PyObject *)v, item); + v = Vec_T_Append(v, item); Py_DECREF(item); - if (v == NULL) { + if (VEC_IS_ERROR(v)) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); return NULL; } } Py_DECREF(iter); if (PyErr_Occurred()) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } - return v; + return Vec_T_Box(v); } -PyObject *Vec_T_Append(PyObject *obj, PyObject *x) { - VecTObject *vec = (VecTObject *)obj; - Py_ssize_t cap = VEC_SIZE(vec); - Py_ssize_t len = vec->len; +VecT Vec_T_Append(VecT vec, PyObject *x) { + Py_ssize_t cap = VEC_CAP(vec); Py_INCREF(x); - if (len < cap) { - vec->items[len] = x; - vec->len = len + 1; - return (PyObject *)vec; + if (vec.len < cap) { + fprintf(stderr, "simple\n"); + vec.buf->items[vec.len] = x; + vec.len++; + return vec; } else { + fprintf(stderr, "grow\n"); Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTObject *new = (VecTObject *)Vec_T_New(new_size, (PyObject *)vec->item_type); - if (new == NULL) - return NULL; - memcpy(new->items, vec->items, sizeof(PyObject *) * len); - memset(vec->items, 0, sizeof(PyObject *) * len); - new->items[len] = x; - new->len = len + 1; - Py_DECREF(vec); - return (PyObject *)new; + VecT new = vec_t_alloc(new_size, vec.buf->item_type); + if (VEC_IS_ERROR(new)) + return new; + // Copy items to new vec. + memcpy(new.buf->items, vec.buf->items, sizeof(PyObject *) * vec.len); + memset(new.buf->items + vec.len, 0, sizeof(PyObject *) * (new_size - vec.len)); + // Clear the items in the old vec. We avoid reference count manipulation. + memset(vec.buf->items, 0, sizeof(PyObject *) * vec.len); + new.buf->items[vec.len] = x; + new.len = vec.len + 1; + VEC_DECREF(vec); + return new; } } VecTFeatures TFeatures = { &VecTType, + &VecbufTType, Vec_T_New, + Vec_T_Box, Vec_T_Append, Vec_T_Pop, Vec_T_Remove, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 16142eab0fb31..7f3252dc2be83 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -24,15 +24,17 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { return NULL; } - return NULL; -#if 0 VecProxy *p = (VecProxy *)self; if (p->optionals == 0 && p->depth == 0) { if (init == NULL) { - return (PyObject *)Vec_T_New(0, (PyObject *)p->item_type); + VecT vec = Vec_T_New(0, (PyObject *)p->item_type); + if (VEC_IS_ERROR(vec)) + return NULL; + return Vec_T_Box(vec); } else { - return (PyObject *)Vec_T_FromIterable((PyTypeObject *)p->item_type, init); + return Vec_T_FromIterable((PyTypeObject *)p->item_type, init); } +#if 0 } else { if (init == NULL) { return (PyObject *)Vec_T_Ext_New(0, (PyObject *)p->item_type, p->optionals, @@ -41,8 +43,9 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) return (PyObject *)Vec_T_Ext_FromIterable((PyTypeObject *)p->item_type, p->optionals, p->depth, init); } - } #endif + } + return NULL; // TODO remove } static int @@ -330,13 +333,17 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return Vec_I64_Box(v); -#if 0 } else if (VecT_Check(vec)) { - VecTObject *v = (VecTObject *)vec; - if (!VecT_ItemCheck(v, item)) + VecT v = ((VecTObject *)vec)->vec; + if (!VecT_ItemCheck(v, item)) { return NULL; - Py_INCREF(vec); - return Vec_T_Append(vec, item); + } + VEC_INCREF(v); + v = Vec_T_Append(v, item); + if (VEC_IS_ERROR(v)) + return NULL; + return Vec_T_Box(v); +#if 0 } else if (VecTExt_Check(vec)) { VecTExtObject *v = (VecTExtObject *)vec; if (!VecTExt_ItemCheck(v, item)) @@ -387,12 +394,17 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecProxyType) < 0) return NULL; -#if 0 + if (PyType_Ready(&VecTType) < 0) return NULL; + if (PyType_Ready(&VecbufTType) < 0) + return NULL; + +#if 0 if (PyType_Ready(&VecTExtType) < 0) return NULL; #endif + if (PyType_Ready(&VecI64Type) < 0) return NULL; if (PyType_Ready(&VecbufI64Type) < 0) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 35ffb4f7c52d4..d61a7a0c49aa8 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -105,12 +105,13 @@ typedef struct _VecI64Features { typedef struct _VecTFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - PyObject *(*alloc)(Py_ssize_t, PyObject *); - PyObject *(*append)(PyObject *, PyObject *); - PyObject *(*pop)(PyObject *, Py_ssize_t); - int (*remove)(PyObject *, PyObject *); + VecT (*alloc)(Py_ssize_t, PyObject *); + PyObject *(*box)(VecT); + VecT (*append)(VecT, PyObject *); + VecT (*pop)(VecT, Py_ssize_t, PyObject **result); + VecT (*remove)(VecT, PyObject *); // TODO: Py_ssize_t - PyObject *(*slice)(PyObject *, int64_t, int64_t); + VecT (*slice)(VecT, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); @@ -141,7 +142,7 @@ typedef struct { VecTExtFeatures *t_ext; } VecCapsule; -#define VEC_SIZE(v) ((v)->ob_base.ob_size) +#define BUF_SIZE(b) ((b)->ob_base.ob_size) #define VEC_CAP(v) ((v).buf->ob_base.ob_size) #define VEC_IS_ERROR(v) ((v).len < 0) #define VEC_DECREF(v) Py_XDECREF((v).buf) @@ -152,6 +153,11 @@ inline VecI64 Vec_I64_Error() { return v; } +inline VecT Vec_T_Error() { + VecT v = { .len = -1 }; + return v; +} + // Type objects extern PyTypeObject VecbufI64Type; @@ -182,10 +188,8 @@ static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; } -#if 0 - -static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { - if (PyObject_TypeCheck(item, v->item_type)) +static inline int VecT_ItemCheck(VecT v, PyObject *item) { + if (PyObject_TypeCheck(item, v.buf->item_type)) return 1; else { // TODO: better error message @@ -194,9 +198,12 @@ static inline int VecT_ItemCheck(VecTObject *v, PyObject *item) { } } -PyObject *Vec_T_New(Py_ssize_t size, PyObject *item_type); -VecTObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable); -PyObject *Vec_T_Append(PyObject *obj, PyObject *x); +VecT Vec_T_New(Py_ssize_t size, PyObject *item_type); +PyObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable); +PyObject *Vec_T_Box(VecT); +VecT Vec_T_Append(VecT vec, PyObject *x); + +#if 0 // vec[t] operations (extended) From 9daeac457a5d849f67968effcbe6258f35da1a2d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 14:20:08 +0100 Subject: [PATCH 065/208] Implement vec[t] remove --- mypyc/lib-rt/vecs/vec_t.c | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index ad71c5d276758..0abac96874426 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -159,20 +159,39 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { } static VecT Vec_T_Remove(VecT v, PyObject *arg) { - return Vec_T_Error(); // TODO implement - //VecTObject *v = (VecTObject *)self; - //return vec_generic_remove(&v->len, v->items, arg); + PyObject **items = v.buf->items; + for (Py_ssize_t i = 0; i < v.len; i++) { + int match = 0; + if (items[i] == arg) + match = 1; + else { + int itemcmp = PyObject_RichCompareBool(items[i], arg, Py_EQ); + if (itemcmp < 0) + return Vec_T_Error(); + match = itemcmp; + } + if (match) { + Py_CLEAR(items[i]); + for (; i < v.len - 1; i++) { + items[i] = items[i + 1]; + } + v.len--; + VEC_INCREF(v); + return v; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + return Vec_T_Error(); } static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { - Py_RETURN_NONE; // TODO implement -#if 0 - VecTObject *v = (VecTObject *)self; + VecT v = ((VecTObject *)self)->vec; if (!VecT_ItemCheck(v, arg)) return NULL; - if (!vec_generic_remove(&v->len, v->items, arg)) + v = Vec_T_Remove(v, arg); + if (VEC_IS_ERROR(v)) return NULL; -#endif + return Vec_T_Box(v); } static VecT Vec_T_Pop(VecT self, Py_ssize_t index, PyObject **result) { @@ -339,12 +358,10 @@ VecT Vec_T_Append(VecT vec, PyObject *x) { Py_ssize_t cap = VEC_CAP(vec); Py_INCREF(x); if (vec.len < cap) { - fprintf(stderr, "simple\n"); vec.buf->items[vec.len] = x; vec.len++; return vec; } else { - fprintf(stderr, "grow\n"); Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here VecT new = vec_t_alloc(new_size, vec.buf->item_type); From 240a668cc5204f8a7ef681fcad21ea78b98d2a37 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 15:22:03 +0100 Subject: [PATCH 066/208] Implement vec[t] pop --- mypyc/lib-rt/vecs/vec_t.c | 51 +++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 0abac96874426..b336aa8b0f538 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -175,6 +175,8 @@ static VecT Vec_T_Remove(VecT v, PyObject *arg) { for (; i < v.len - 1; i++) { items[i] = items[i + 1]; } + if (v.len > 0) + Py_XINCREF(items[v.len - 1]); v.len--; VEC_INCREF(v); return v; @@ -194,16 +196,51 @@ static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { return Vec_T_Box(v); } -static VecT Vec_T_Pop(VecT self, Py_ssize_t index, PyObject **result) { - return Vec_T_Error(); // TODO implement - //VecTObject *v = (VecTObject *)self; - //return vec_generic_pop(&v->len, v->items, index); +static VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { + if (index < 0) + index += v.len; + + if (index < 0 || index >= v.len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return Vec_T_Error(); + } + + PyObject **items = v.buf->items; + *result = items[index]; + for (Py_ssize_t i = index; i < v.len - 1; i++) + items[i] = items[i + 1]; + if (v.len > 0) + Py_XINCREF(items[v.len - 1]); + v.len--; + VEC_INCREF(v); + return v; } static PyObject *vec_t_pop(PyObject *self, PyObject *args) { - return NULL; // TODO implement - //VecTObject *v = (VecTObject *)self; - //return vec_generic_pop_wrapper(&v->len, v->items, args); + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; + + VecT v = ((VecTObject *)self)->vec; + PyObject *result; + v = Vec_T_Pop(v, index, &result); + if (VEC_IS_ERROR(v)) + return NULL; + + PyObject *vboxed = Vec_T_Box(v); + if (vboxed == NULL) + return NULL; + + PyObject *res = PyTuple_New(2); + if (res == NULL) { + Py_DECREF(vboxed); + Py_DECREF(result); + return NULL; + } + + PyTuple_SetItem(res, 0, vboxed); + PyTuple_SetItem(res, 1, result); + return res; } static int From 3164fd40c457ec5b2f7f5dbbbf508e4ae786d6d2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 16:38:10 +0100 Subject: [PATCH 067/208] Work towards vec[t | None] support There is still some memory issue going on. --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 24 ++++++++++++++---------- mypyc/lib-rt/vecs/vecs.c | 26 ++++++++++++++++---------- mypyc/lib-rt/vecs/vecs.h | 23 ++++++++++++++--------- 4 files changed, 45 insertions(+), 30 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 9c60f9b3260f2..d7ac79d1fe863 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -91,7 +91,7 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } PyObject *vec_i64_repr(PyObject *self) { - return vec_repr(self, I64TypeObj, 0, 0, 1); + return vec_repr(self, (size_t)I64TypeObj, 0, 0, 1); } PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index b336aa8b0f538..6c7c23e71d86b 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -11,12 +11,12 @@ #include "vecs.h" // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. -static VecT vec_t_alloc(Py_ssize_t size, PyTypeObject *item_type) { +static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { VecbufTObject *buf = PyObject_GC_NewVar(VecbufTObject, &VecbufTType, size); if (buf == NULL) return Vec_T_Error(); - Py_INCREF(item_type); buf->item_type = item_type; + Py_INCREF(BUF_ITEM_TYPE(buf)); VecT res = { .buf = buf }; PyObject_GC_Track(buf); return res; @@ -33,8 +33,8 @@ PyObject *Vec_T_Box(VecT vec) { return (PyObject *)obj; } -VecT Vec_T_New(Py_ssize_t size, PyObject *item_type) { - VecT vec = vec_t_alloc(size, (PyTypeObject *)item_type); +VecT Vec_T_New(Py_ssize_t size, size_t item_type) { + VecT vec = vec_t_alloc(size, item_type); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < size; i++) { @@ -253,7 +253,7 @@ VecT_traverse(VecTObject *self, visitproc visit, void *arg) static int VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) { - Py_VISIT(self->item_type); + Py_VISIT(BUF_ITEM_TYPE(self)); for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_VISIT(self->items[i]); } @@ -270,7 +270,10 @@ VecT_clear(VecTObject *self) static int VecbufT_clear(VecbufTObject *self) { - Py_CLEAR(self->item_type); + if (self->item_type) { + Py_DECREF(BUF_ITEM_TYPE(self)); + self->item_type = 0; + } for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_CLEAR(self->items[i]); } @@ -292,7 +295,10 @@ VecbufT_dealloc(VecbufTObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecbufT_dealloc) - Py_CLEAR(self->item_type); + if (self->item_type) { + Py_DECREF(BUF_ITEM_TYPE(self)); + self->item_type = 0; + } for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_CLEAR(self->items[i]); } @@ -354,13 +360,11 @@ PyTypeObject VecTType = { // TODO: free }; -PyObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable) { +PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { VecT v = vec_t_alloc(0, item_type); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; - Py_INCREF(item_type); - v.buf->item_type = item_type; PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 7f3252dc2be83..007386a545af2 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -12,7 +12,8 @@ PyTypeObject *I64TypeObj; // These aren't really types. This only supports constructing instances. typedef struct { PyObject_HEAD - PyTypeObject *item_type; + // Tagged pointer to PyTypeObject *, lowest bit set for optional item type + size_t item_type; int32_t depth; // Number of nested VecTExt or VecT types int32_t optionals; // Flags for optional types on each nesting level } VecProxy; @@ -25,14 +26,14 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) return NULL; } VecProxy *p = (VecProxy *)self; - if (p->optionals == 0 && p->depth == 0) { + if (p->depth == 0) { if (init == NULL) { - VecT vec = Vec_T_New(0, (PyObject *)p->item_type); + VecT vec = Vec_T_New(0, p->item_type); if (VEC_IS_ERROR(vec)) return NULL; return Vec_T_Box(vec); } else { - return Vec_T_FromIterable((PyTypeObject *)p->item_type, init); + return Vec_T_FromIterable(p->item_type, init); } #if 0 } else { @@ -58,18 +59,22 @@ VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) static void VecProxy_dealloc(VecProxy *self) { - Py_CLEAR(self->item_type); + if (self->item_type) { + Py_DECREF((PyObject *)(self->item_type & ~1)); + self->item_type = 0; + } PyObject_GC_Del(self); } -PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals) { +PyObject *vec_type_to_str(size_t item_type, int32_t depth, int32_t optionals) { PyObject *item; if (depth == 0) - item = PyObject_GetAttrString((PyObject *)item_type, "__name__"); + item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); else item = vec_type_to_str(item_type, depth - 1, optionals >> 1); - if (optionals & 1) { + // TODO: clean up + if ((optionals & 1) || (item_type & 1)) { item = PyUnicode_FromFormat("%U | None", item); } return PyUnicode_FromFormat("vec[%U]", item); @@ -168,7 +173,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (p == NULL) return NULL; Py_INCREF(item); - p->item_type = (PyTypeObject *)item; + p->item_type = (size_t)item | (optionals & 1); p->depth = depth; p->optionals = optionals; PyObject_GC_Track(p); @@ -190,8 +195,9 @@ PyTypeObject VecGenericType = { .tp_methods = vec_methods, }; -PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, +PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optionals, int verbose) { + // TODO: Check for errors PyObject *l = Py_BuildValue("[]"); PyObject *prefix; PyObject *mid; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index d61a7a0c49aa8..24b8b080abeee 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -21,14 +21,16 @@ typedef struct _VecbufI64Object { // Simple generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecbufTObject { PyObject_VAR_HEAD - PyTypeObject *item_type; + // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. + size_t item_type; PyObject *items[1]; } VecbufTObject; // Extended generic vec type: vec[t | None], vec[vec[...]], etc. typedef struct _VecbufTExtObject { PyObject_VAR_HEAD - PyTypeObject *item_type; + // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. + size_t item_type; int32_t depth; // Number of nested VecTExt or VecT types int32_t optionals; // Flags for optional types on each nesting level PyObject *items[1]; @@ -105,7 +107,7 @@ typedef struct _VecI64Features { typedef struct _VecTFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecT (*alloc)(Py_ssize_t, PyObject *); + VecT (*alloc)(Py_ssize_t, size_t); PyObject *(*box)(VecT); VecT (*append)(VecT, PyObject *); VecT (*pop)(VecT, Py_ssize_t, PyObject **result); @@ -143,6 +145,7 @@ typedef struct { } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) +#define BUF_ITEM_TYPE(b) ((PyTypeObject *)((b)->item_type & ~1)) #define VEC_CAP(v) ((v).buf->ob_base.ob_size) #define VEC_IS_ERROR(v) ((v).len < 0) #define VEC_DECREF(v) Py_XDECREF((v).buf) @@ -189,17 +192,19 @@ static inline int VecT_Check(PyObject *o) { } static inline int VecT_ItemCheck(VecT v, PyObject *item) { - if (PyObject_TypeCheck(item, v.buf->item_type)) + if (PyObject_TypeCheck(item, BUF_ITEM_TYPE(v.buf))) { return 1; - else { + } else if ((v.buf->item_type & 1) && item == Py_None) { + return 1; + } else { // TODO: better error message PyErr_SetString(PyExc_TypeError, "invalid item type"); return 0; } } -VecT Vec_T_New(Py_ssize_t size, PyObject *item_type); -PyObject *Vec_T_FromIterable(PyTypeObject *item_type, PyObject *iterable); +VecT Vec_T_New(Py_ssize_t size, size_t item_type); +PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT); VecT Vec_T_Append(VecT vec, PyObject *x); @@ -247,8 +252,8 @@ static inline int check_float_error(PyObject *o) { return 0; } -PyObject *vec_type_to_str(PyTypeObject *item_type, int32_t depth, int32_t optionals); -PyObject *vec_repr(PyObject *vec, PyTypeObject *item_type, int32_t depth, int32_t optionals, +PyObject *vec_type_to_str(size_t item_type, int32_t depth, int32_t optionals); +PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optionals, int verbose); PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, From a50736ca9105775faa0ce7ebfcd41f309373f7ca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 16:52:38 +0100 Subject: [PATCH 068/208] Fix crash --- mypyc/lib-rt/vecs/vecs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 007386a545af2..90267822165d4 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -52,7 +52,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) static int VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) { - Py_VISIT(self->item_type); + Py_VISIT((PyObject *)(self->item_type & ~1)); return 0; } From 4c78b0ce71ae387a06b6bd40351f1ed60392a6c7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 17:54:26 +0100 Subject: [PATCH 069/208] Update header to use new repr for nested vecs --- mypyc/lib-rt/vecs/vecs.h | 50 ++++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 24b8b080abeee..8d7b941da3cfa 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -26,6 +26,11 @@ typedef struct _VecbufTObject { PyObject *items[1]; } VecbufTObject; +typedef struct _VecbufTExtItem { + Py_ssize_t len; + PyObject *buf; +} VecbufTExtItem; + // Extended generic vec type: vec[t | None], vec[vec[...]], etc. typedef struct _VecbufTExtObject { PyObject_VAR_HEAD @@ -33,7 +38,7 @@ typedef struct _VecbufTExtObject { size_t item_type; int32_t depth; // Number of nested VecTExt or VecT types int32_t optionals; // Flags for optional types on each nesting level - PyObject *items[1]; + VecbufTExtItem items[1]; } VecbufTExtObject; @@ -126,12 +131,12 @@ typedef struct _VecTFeatures { typedef struct _VecTExtFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - PyObject *(*alloc)(Py_ssize_t, PyObject *, int optionals, int depth); - PyObject *(*append)(PyObject *, PyObject *); - PyObject *(*pop)(PyObject *, Py_ssize_t); - int (*remove)(PyObject *, PyObject *); + VecTExt (*alloc)(Py_ssize_t, size_t, int optionals, int depth); + VecTExt (*append)(VecTExt, PyObject *); + VecTExt (*pop)(VecTExt, Py_ssize_t, PyObject **result); + VecTExt (*remove)(VecTExt, PyObject *); // TODO: Py_ssize_t - PyObject *(*slice)(PyObject *, int64_t, int64_t); + VecTExt (*slice)(VecTExt, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); @@ -161,6 +166,11 @@ inline VecT Vec_T_Error() { return v; } +inline VecTExt Vec_T_Ext_Error() { + VecTExt v = { .len = -1 }; + return v; +} + // Type objects extern PyTypeObject VecbufI64Type; @@ -208,26 +218,22 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT); VecT Vec_T_Append(VecT vec, PyObject *x); -#if 0 - // vec[t] operations (extended) static inline int VecTExt_Check(PyObject *o) { return o->ob_type == &VecTExtType; } -static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { - if (v->depth == 0 && PyObject_TypeCheck(it, v->item_type)) { +static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { + if (it == Py_None && (v.buf->optionals & 1)) { return 1; - } else if (it == Py_None && (v->optionals & 1)) { - return 1; - } else if (v->depth == 1 && it->ob_type == &VecTType - && ((VecTExtObject *)it)->item_type == v->item_type) { + } else if (v.buf->depth == 1 && it->ob_type == &VecTType + && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type) { return 1; } else if (it->ob_type == &VecTExtType - && ((VecTExtObject *)it)->depth == v->depth - 1 - && ((VecTExtObject *)it)->item_type == v->item_type - && ((VecTExtObject *)it)->optionals == (v->optionals >> 1)) { + && ((VecTExtObject *)it)->vec.buf->depth == v.buf->depth - 1 + && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type + && ((VecTExtObject *)it)->vec.buf->optionals == (v.buf->optionals >> 1)) { return 1; } else { // TODO: better error message @@ -236,12 +242,12 @@ static inline int VecTExt_ItemCheck(VecTExtObject *v, PyObject *it) { } } -#endif +VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth); +PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t depth, + PyObject *iterable); +PyObject *Vec_T_Ext_Box(VecTExt); +VecTExt Vec_T_Ext_Append(VecTExt vec, PyObject *x); -PyObject *Vec_T_Ext_New(Py_ssize_t size, PyObject *item_type, int32_t optionals, int32_t depth); -VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, - PyObject *iterable); -PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x); // Misc helpers static inline int check_float_error(PyObject *o) { From 6c76516c0f217c8ef0df8f95038c38020d48cd0e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 15 Apr 2023 18:17:04 +0100 Subject: [PATCH 070/208] Work towards nested vecs (still not working!) --- mypyc/lib-rt/vecs/vec_t.c | 2 - mypyc/lib-rt/vecs/vec_t_ext.c | 142 ++++++++++++++++++++++------------ mypyc/lib-rt/vecs/vecs.c | 8 +- 3 files changed, 95 insertions(+), 57 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 6c7c23e71d86b..7989cdf7533f3 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -22,8 +22,6 @@ static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { return res; } -// TODO: vec_t_dealloc - PyObject *Vec_T_Box(VecT vec) { VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); if (obj == NULL) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 5ce5c74c49dd2..59dbad2e446e1 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,83 +9,105 @@ #include #include "vecs.h" -// Alloc a partially initialized vec. Caller *must* initialize len and items, and -// call PyObject_GC_Track(). -static VecTExtObject *vec_t_ext_alloc(Py_ssize_t size, PyTypeObject *item_type, int32_t optionals, - int32_t depth) { - VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); - if (v == NULL) - return NULL; - Py_INCREF(item_type); - v->item_type = item_type; - v->optionals = optionals; - v->depth = depth; - return v; +// Alloc a partially initialized vec. Caller *must* initialize len and buf->items. +static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, + int32_t depth) { + VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); + if (buf == NULL) + return Vec_T_Ext_Error(); + buf->item_type = item_type; + buf->optionals = optionals; + buf->depth = depth; + Py_INCREF(BUF_ITEM_TYPE(buf)); + VecTExt res = { .buf = buf }; + PyObject_GC_Track(buf); + return res; } -PyObject *Vec_T_Ext_New(Py_ssize_t size, PyObject *item_type, int32_t optionals, int32_t depth) { - VecTExtObject *v; - v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, size); - //v = VecTType.tp_alloc(&VecTType, size); - if (v == NULL) +PyObject *Vec_T_Ext_Box(VecTExt vec) { + VecTExtObject *obj = PyObject_GC_New(VecTExtObject, &VecTExtType); + if (obj == NULL) return NULL; + obj->vec = vec; + PyObject_GC_Track(obj); + return (PyObject *)obj; +} - Py_INCREF(item_type); - v->item_type = (PyTypeObject *)item_type; - v->len = size; +VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { + VecTExt vec = vec_t_ext_alloc(size, item_type, optionals, depth); + if (VEC_IS_ERROR(vec)) + return vec; for (Py_ssize_t i = 0; i < size; i++) { - v->items[i] = NULL; + vec.buf->items[i].len = -1; + vec.buf->items[i].buf = NULL; } - v->optionals = optionals; - v->depth = depth; - - PyObject_GC_Track(v); - return (PyObject *)v; + vec.len = size; + return vec; } PyObject *vec_t_ext_repr(PyObject *self) { - VecTExtObject *v = (VecTExtObject *)self; - return vec_repr(self, v->item_type, v->depth, v->optionals, 1); + VecTExt v = ((VecTExtObject *)self)->vec; + return vec_repr(self, v.buf->item_type, v.buf->depth, v.buf->optionals, 1); } PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { - VecTExtObject *v = (VecTExtObject *)o; - if ((size_t)i < (size_t)v->len) { - PyObject *item = v->items[i]; - Py_INCREF(item); - return item; + VecTExt v = ((VecTExtObject *)o)->vec; + if ((size_t)i < (size_t)v.len) { + VecbufTExtItem item = v.buf->items[i]; + if (item.len < 0) + Py_RETURN_NONE; + else if (v.buf->depth > 1) { + // Item is a nested vec + VecTExt item = { .len = item.len, .buf = item.buf }; + return Vec_T_Ext_Box(item); + } else { + // Item is a non-nested vec + void *item_type = (void *)(v.buf->item_type & ~1); + if (item_type == I64TypeObj) { + // vec[i64] + VecI64 item = { .len = item.len, .buf = item.buf }; + return Vec_I64_Box(item); + } else { + // Generic vec[t] + VecT item = { .len = item.len, .buf = item.buf }; + return Vec_T_Box(item); + } + } } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; } } -PyObject *Vec_T_Ext_Slice(PyObject *self, int64_t start, int64_t end) { - VecTExtObject *vec = (VecTExtObject *)self; +VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { if (start < 0) - start += vec->len; + start += vec.len; if (end < 0) - end += vec->len; + end += vec.len; if (end < start) end = start; if (start < 0) start = 0; - if (end > vec->len) - end = vec->len; + if (end > vec.len) + end = vec.len; int64_t slicelength = end - start; - VecTExtObject *res = vec_t_ext_alloc(slicelength, vec->item_type, vec->optionals, vec->depth); - if (res == NULL) - return NULL; - res->len = slicelength; + VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->optionals, + vec.buf->depth); + if (VEC_IS_ERROR(res)) + return res; + res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec->items[start + i]; - Py_INCREF(item); - res->items[i] = item; + VecbufTExtItem item = vec.buf->items[start + i]; + Py_INCREF(item.buf); + res.buf->items[i] = item; } - return (PyObject *)res; + return res; } +#if 0 + PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { + // TODO VecTExtObject *vec = (VecTExtObject *)self; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -126,6 +148,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { } int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + // TODO VecTExtObject *v = (VecTExtObject *)self; if (!VecTExt_ItemCheck(v, o)) return -1; @@ -140,6 +163,7 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { + // TODO PyObject *res; if (op == Py_EQ || op == Py_NE) { if (other->ob_type != &VecTExtType) { @@ -161,11 +185,13 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { } static int Vec_T_Ext_Remove(PyObject *self, PyObject *arg) { + // TODO VecTExtObject *v = (VecTExtObject *)self; return vec_generic_remove(&v->len, v->items, arg); } static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { + // TODO VecTExtObject *v = (VecTExtObject *)self; if (!VecTExt_ItemCheck(v, arg)) return NULL; @@ -174,12 +200,13 @@ static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { Py_RETURN_NONE; } -static PyObject *Vec_T_Ext_Pop(PyObject *self, Py_ssize_t index) { - VecTExtObject *v = (VecTExtObject *)self; - return vec_generic_pop(&v->len, v->items, index); +static VecTExt Vec_T_Ext_Pop(VecTExt vec, Py_ssize_t index, PyObject **result) { + // TODO + return Vec_T_Ext_Error(); } static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { + // TODO VecTExtObject *v = (VecTExtObject *)self; return vec_generic_pop_wrapper(&v->len, v->items, args); } @@ -187,6 +214,7 @@ static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { static int VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) { + // TODO Py_VISIT(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { Py_VISIT(self->items[i]); @@ -197,6 +225,7 @@ VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) static int VecTExt_clear(VecTExtObject *self) { + // TODO Py_CLEAR(self->item_type); for (Py_ssize_t i = 0; i < self->len; i++) { Py_CLEAR(self->items[i]); @@ -207,6 +236,7 @@ VecTExt_clear(VecTExtObject *self) static void VecTExt_dealloc(VecTExtObject *self) { + // TODO PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) Py_XDECREF(self->item_type); @@ -217,9 +247,11 @@ VecTExt_dealloc(VecTExtObject *self) Py_TRASHCAN_END } +#endif + static Py_ssize_t vec_ext_length(PyObject *o) { // TODO: Type check o - return ((VecTExtObject *)o)->len; + return ((VecTExtObject *)o)->vec.len; } static PyMappingMethods VecTExtMapping = { @@ -257,8 +289,11 @@ PyTypeObject VecTExtType = { // TODO: free }; +#if 0 + VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, PyObject *iterable) { + // TODO VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, 0); if (v == NULL) return NULL; @@ -299,6 +334,7 @@ VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals } PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { + // TODO VecTExtObject *vec = (VecTExtObject *)obj; Py_ssize_t cap = VEC_SIZE(vec); Py_ssize_t len = vec->len; @@ -325,9 +361,13 @@ PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { VecTExtFeatures TExtFeatures = { &VecTExtType, + &VecbufTExtType, Vec_T_Ext_New, + Vec_T_Ext_Box, Vec_T_Ext_Append, Vec_T_Ext_Pop, Vec_T_Ext_Remove, Vec_T_Ext_Slice, }; + +#endif diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 90267822165d4..1e8b63d094d25 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -378,8 +378,8 @@ static PyModuleDef vecsmodule = { static VecCapsule Capsule = { &I64Features, - NULL, // TODO: &TFeatures, - NULL, // TODO: &TExtFeatures, + &TFeatures, + &TExtFeatures, }; PyMODINIT_FUNC @@ -406,10 +406,10 @@ PyInit_vecs(void) if (PyType_Ready(&VecbufTType) < 0) return NULL; -#if 0 if (PyType_Ready(&VecTExtType) < 0) return NULL; -#endif + if (PyType_Ready(&VecbufTExtType) < 0) + return NULL; if (PyType_Ready(&VecI64Type) < 0) return NULL; From 1656992eab7dd1eb8b2760af02ccb8fdafd94c90 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 09:56:19 +0100 Subject: [PATCH 071/208] Implement nested vec subscription + refactor --- mypyc/lib-rt/vecs/vec_t_ext.c | 77 ++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 59dbad2e446e1..be87d3ee5c497 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -50,29 +50,34 @@ PyObject *vec_t_ext_repr(PyObject *self) { return vec_repr(self, v.buf->item_type, v.buf->depth, v.buf->optionals, 1); } +static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { + VecbufTExtItem item = v.buf->items[index]; + if (item.len < 0) + Py_RETURN_NONE; + Py_INCREF(item.buf); + if (v.buf->depth > 1) { + // Item is a nested vec + VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; + return Vec_T_Ext_Box(v); + } else { + // Item is a non-nested vec + void *item_type = (void *)(v.buf->item_type & ~1); + if (item_type == I64TypeObj) { + // vec[i64] + VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; + return Vec_I64_Box(v); + } else { + // Generic vec[t] + VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; + return Vec_T_Box(v); + } + } +} + PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { VecTExt v = ((VecTExtObject *)o)->vec; if ((size_t)i < (size_t)v.len) { - VecbufTExtItem item = v.buf->items[i]; - if (item.len < 0) - Py_RETURN_NONE; - else if (v.buf->depth > 1) { - // Item is a nested vec - VecTExt item = { .len = item.len, .buf = item.buf }; - return Vec_T_Ext_Box(item); - } else { - // Item is a non-nested vec - void *item_type = (void *)(v.buf->item_type & ~1); - if (item_type == I64TypeObj) { - // vec[i64] - VecI64 item = { .len = item.len, .buf = item.buf }; - return Vec_I64_Box(item); - } else { - // Generic vec[t] - VecT item = { .len = item.len, .buf = item.buf }; - return Vec_T_Box(item); - } - } + return box_vec_item(v, i); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -104,19 +109,14 @@ VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { return res; } -#if 0 - PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { - // TODO - VecTExtObject *vec = (VecTExtObject *)self; + VecTExt vec = ((VecTExtObject *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) return NULL; - if ((size_t)i < (size_t)vec->len) { - PyObject *item = vec->items[i]; - Py_INCREF(item); - return item; + if ((size_t)i < (size_t)vec.len) { + return box_vec_item(vec, i); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -125,21 +125,20 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec->len, &start, &stop, step); - VecTExtObject *res = vec_t_ext_alloc(slicelength, vec->item_type, vec->optionals, - vec->depth); - if (res == NULL) + Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); + VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->optionals, + vec.buf->depth); + if (VEC_IS_ERROR(res)) return NULL; - res->len = slicelength; + res.len = slicelength; Py_ssize_t j = start; for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec->items[j]; - Py_INCREF(item); - res->items[i] = item; + VecbufTExtItem item = vec.buf->items[j]; + Py_INCREF(item.buf); + res.buf->items[i] = item; j += step; } - PyObject_GC_Track(res); - return (PyObject *)res; + return Vec_T_Ext_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -147,6 +146,8 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { } } +#if 0 + int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { // TODO VecTExtObject *v = (VecTExtObject *)self; From e2b87aa4b246c49de5c3c7da3715517aee962f41 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 10:07:57 +0100 Subject: [PATCH 072/208] Implement more nested vec features --- mypyc/lib-rt/vecs/vec_t.c | 34 +++++++-------- mypyc/lib-rt/vecs/vec_t_ext.c | 79 ++++++++++++++++++++++++++--------- mypyc/lib-rt/vecs/vecs.h | 1 + 3 files changed, 78 insertions(+), 36 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 7989cdf7533f3..74e354a6d3b37 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -249,19 +249,29 @@ VecT_traverse(VecTObject *self, visitproc visit, void *arg) } static int -VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) +VecT_clear(VecTObject *self) { - Py_VISIT(BUF_ITEM_TYPE(self)); - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { - Py_VISIT(self->items[i]); - } + Py_CLEAR(self->vec.buf); return 0; } -static int -VecT_clear(VecTObject *self) +static void +VecT_dealloc(VecTObject *self) { + PyObject_GC_UnTrack(self); + Py_TRASHCAN_BEGIN(self, VecT_dealloc) Py_CLEAR(self->vec.buf); + Py_TYPE(self)->tp_free((PyObject *)self); + Py_TRASHCAN_END +} + +static int +VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) +{ + Py_VISIT(BUF_ITEM_TYPE(self)); + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_VISIT(self->items[i]); + } return 0; } @@ -278,16 +288,6 @@ VecbufT_clear(VecbufTObject *self) return 0; } -static void -VecT_dealloc(VecTObject *self) -{ - PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecT_dealloc) - Py_CLEAR(self->vec.buf); - Py_TYPE(self)->tp_free((PyObject *)self); - Py_TRASHCAN_END -} - static void VecbufT_dealloc(VecbufTObject *self) { diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index be87d3ee5c497..25a4714066ef2 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -150,7 +150,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { // TODO - VecTExtObject *v = (VecTExtObject *)self; + VecTExt v = ((VecTExtObject *)self)->vec; if (!VecTExt_ItemCheck(v, o)) return -1; if ((size_t)i < (size_t)v->len) { @@ -212,44 +212,71 @@ static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { return vec_generic_pop_wrapper(&v->len, v->items, args); } +#endif + static int -VecTExt_traverse(VecTExtObject *self, visitproc visit, void *arg) +VecTExt_traverse(VecTObject *self, visitproc visit, void *arg) { - // TODO - Py_VISIT(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { - Py_VISIT(self->items[i]); + Py_VISIT(self->vec.buf); + return 0; +} + +static int +VecTExt_clear(VecTObject *self) +{ + Py_CLEAR(self->vec.buf); + return 0; +} + +static void +VecTExt_dealloc(VecTObject *self) +{ + PyObject_GC_UnTrack(self); + Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) + Py_CLEAR(self->vec.buf); + Py_TYPE(self)->tp_free((PyObject *)self); + Py_TRASHCAN_END +} + +static int +VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) +{ + Py_VISIT(BUF_ITEM_TYPE(self)); + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_VISIT(self->items[i].buf); } return 0; } static int -VecTExt_clear(VecTExtObject *self) +VecbufTExt_clear(VecbufTExtObject *self) { - // TODO - Py_CLEAR(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { - Py_CLEAR(self->items[i]); + if (self->item_type) { + Py_DECREF(BUF_ITEM_TYPE(self)); + self->item_type = 0; + } + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_CLEAR(self->items[i].buf); } return 0; } static void -VecTExt_dealloc(VecTExtObject *self) +VecbufTExt_dealloc(VecbufTExtObject *self) { - // TODO PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) - Py_XDECREF(self->item_type); - for (Py_ssize_t i = 0; i < self->len; i++) { - Py_XDECREF(self->items[i]); + Py_TRASHCAN_BEGIN(self, VecbufTExt_dealloc) + if (self->item_type) { + Py_DECREF(BUF_ITEM_TYPE(self)); + self->item_type = 0; + } + for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_CLEAR(self->items[i].buf); } Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END } -#endif - static Py_ssize_t vec_ext_length(PyObject *o) { // TODO: Type check o return ((VecTExtObject *)o)->vec.len; @@ -271,6 +298,20 @@ static PyMethodDef vec_t_ext_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; +PyTypeObject VecbufTExtType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecbuf", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecbufTExtObject) - sizeof(VecbufTExtItem), + .tp_itemsize = sizeof(VecbufTExtItem), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = (traverseproc)VecbufTExt_traverse, + //.tp_new = vecbuf_i64_new, //?? + .tp_free = PyObject_GC_Del, + .tp_clear = (inquiry)VecbufTExt_clear, + .tp_dealloc = (destructor)VecbufTExt_dealloc, +}; + PyTypeObject VecTExtType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 8d7b941da3cfa..f57338a038b08 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -225,6 +225,7 @@ static inline int VecTExt_Check(PyObject *o) { } static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { + // TODO: vec[i64] item type if (it == Py_None && (v.buf->optionals & 1)) { return 1; } else if (v.buf->depth == 1 && it->ob_type == &VecTType From 9c46dd0a197061866f73f082451d59622b3f6444 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 10:08:43 +0100 Subject: [PATCH 073/208] Minor refactoring --- mypyc/lib-rt/vecs/vec_t_ext.c | 48 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 25a4714066ef2..3fe8ac74b78f1 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,6 +9,30 @@ #include #include "vecs.h" +static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { + VecbufTExtItem item = v.buf->items[index]; + if (item.len < 0) + Py_RETURN_NONE; + Py_INCREF(item.buf); + if (v.buf->depth > 1) { + // Item is a nested vec + VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; + return Vec_T_Ext_Box(v); + } else { + // Item is a non-nested vec + void *item_type = (void *)(v.buf->item_type & ~1); + if (item_type == I64TypeObj) { + // vec[i64] + VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; + return Vec_I64_Box(v); + } else { + // Generic vec[t] + VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; + return Vec_T_Box(v); + } + } +} + // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { @@ -50,30 +74,6 @@ PyObject *vec_t_ext_repr(PyObject *self) { return vec_repr(self, v.buf->item_type, v.buf->depth, v.buf->optionals, 1); } -static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { - VecbufTExtItem item = v.buf->items[index]; - if (item.len < 0) - Py_RETURN_NONE; - Py_INCREF(item.buf); - if (v.buf->depth > 1) { - // Item is a nested vec - VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; - return Vec_T_Ext_Box(v); - } else { - // Item is a non-nested vec - void *item_type = (void *)(v.buf->item_type & ~1); - if (item_type == I64TypeObj) { - // vec[i64] - VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; - return Vec_I64_Box(v); - } else { - // Generic vec[t] - VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; - return Vec_T_Box(v); - } - } -} - PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { VecTExt v = ((VecTExtObject *)o)->vec; if ((size_t)i < (size_t)v.len) { From 823b868dc5b37fea1499f5d6515f82c32d15b1c4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 10:20:55 +0100 Subject: [PATCH 074/208] Add nested vec item unboxing helper --- mypyc/lib-rt/vecs/vec_t_ext.c | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 3fe8ac74b78f1..6befafa89867e 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -33,6 +33,47 @@ static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { } } +// Return 0 on success, -1 on error +static inline int unbox_vec_item(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { + int optionals = v.buf->optionals; + if (item == Py_None && (optionals & 1)) { + unboxed->len = -1; + return 0; + } + int depth = v.buf->depth; + if (depth == 1) { + // TODO: vec[i64] + if (item->ob_type == &VecTType) { + VecTExtObject *o = (VecTExtObject *)item; + if (o->vec.buf->item_type == v.buf->item_type) { + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 1; + } + } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { + VecI64Object *o = (VecI64Object *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 1; + } + } else if (item->ob_type == &VecTExtType) { + VecTExtObject *o = (VecTExtObject *)item; + if (o->vec.buf->depth == v.buf->depth - 1 + && o->vec.buf->item_type == v.buf->item_type + && o->vec.buf->optionals == (optionals >> 1)) { + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 1; + } + } + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return 0; +} + // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { From 0ed34353a5424dfedb920e293e7620ff976fabc2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 10:29:03 +0100 Subject: [PATCH 075/208] Implement nested vec conversion from iterable --- mypyc/lib-rt/vecs/vec_t_ext.c | 37 +++++++++++++++-------------------- mypyc/lib-rt/vecs/vecs.h | 2 +- 2 files changed, 17 insertions(+), 22 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 6befafa89867e..9e874f390ca9f 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -372,50 +372,45 @@ PyTypeObject VecTExtType = { // TODO: free }; -#if 0 - -VecTExtObject *Vec_T_Ext_FromIterable(PyTypeObject *item_type, int32_t optionals, int32_t depth, - PyObject *iterable) { - // TODO - VecTExtObject *v = PyObject_GC_NewVar(VecTExtObject, &VecTExtType, 0); - if (v == NULL) +PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t depth, + PyObject *iterable) { + VecTExt v = vec_t_ext_alloc(0, item_type, optionals, depth); + if (VEC_IS_ERROR(v)) return NULL; - v->len = 0; - Py_INCREF(item_type); - v->item_type = item_type; - v->optionals = optionals; - v->depth = depth; - PyObject_GC_Track(v); + v.len = 0; PyObject *iter = PyObject_GetIter(iterable); if (iter == NULL) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { - if (!VecTExt_ItemCheck(v, item)) { + VecbufTExtItem vecitem; + if (unbox_vec_item(v, item, &vecitem) < 0) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = (VecTExtObject *)Vec_T_Ext_Append((PyObject *)v, item); + v = Vec_T_Ext_Append(v, vecitem); Py_DECREF(item); - if (v == NULL) { + if (VEC_IS_ERROR(v)) { Py_DECREF(iter); - Py_DECREF(v); + VEC_DECREF(v); return NULL; } } Py_DECREF(iter); if (PyErr_Occurred()) { - Py_DECREF(v); + VEC_DECREF(v); return NULL; } - return v; + return Vec_T_Ext_Box(v); } +#if 0 + PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { // TODO VecTExtObject *vec = (VecTExtObject *)obj; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index f57338a038b08..dec3cc1abc063 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -247,7 +247,7 @@ VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int3 PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t depth, PyObject *iterable); PyObject *Vec_T_Ext_Box(VecTExt); -VecTExt Vec_T_Ext_Append(VecTExt vec, PyObject *x); +VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); // Misc helpers From 9e8c5eccb9768c9f359e8876d2e5f1d703b6d9d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 11:58:48 +0100 Subject: [PATCH 076/208] Implement low-level append for nested vecs --- mypyc/lib-rt/vecs/vec_t_ext.c | 45 ++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 9e874f390ca9f..95e89b7c9d6c7 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -409,34 +409,35 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de return Vec_T_Ext_Box(v); } -#if 0 - -PyObject *Vec_T_Ext_Append(PyObject *obj, PyObject *x) { - // TODO - VecTExtObject *vec = (VecTExtObject *)obj; - Py_ssize_t cap = VEC_SIZE(vec); - Py_ssize_t len = vec->len; - Py_INCREF(x); - if (len < cap) { - vec->items[len] = x; - vec->len = len + 1; - return (PyObject *)vec; +VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { + Py_ssize_t cap = VEC_CAP(vec); + Py_INCREF(x.buf); + if (vec.len < cap) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTExtObject *new = (VecTExtObject *)Vec_T_Ext_New( - new_size, (PyObject *)vec->item_type, vec->optionals, vec->depth); - if (new == NULL) - return NULL; - memcpy(new->items, vec->items, sizeof(PyObject *) * len); - memset(vec->items, 0, sizeof(PyObject *) * len); - new->items[len] = x; - new->len = len + 1; - Py_DECREF(vec); - return (PyObject *)new; + VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->optionals, + vec.buf->depth); + if (VEC_IS_ERROR(new)) + return new; + // Copy items to new vec. + memcpy(new.buf->items, vec.buf->items, sizeof(VecbufTExtItem) * vec.len); + // TODO: How to safely represent deleted items? + memset(new.buf->items + vec.len, 0, sizeof(VecbufTExtItem) * (new_size - vec.len)); + // Clear the items in the old vec. We avoid reference count manipulation. + memset(vec.buf->items, 0, sizeof(VecbufTExtItem) * vec.len); + new.buf->items[vec.len] = x; + new.len = vec.len + 1; + VEC_DECREF(vec); + return new; } } +#if 0 + VecTExtFeatures TExtFeatures = { &VecTExtType, &VecbufTExtType, From cca673f7b95fc6aae014db34194fbee7520463f5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 12:03:54 +0100 Subject: [PATCH 077/208] Implement nested vec assignment --- mypyc/lib-rt/vecs/vec_t_ext.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 95e89b7c9d6c7..7c2b2cb89d3fe 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -33,7 +33,8 @@ static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { } } -// Return 0 on success, -1 on error +// Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. +// Return a new reference. static inline int unbox_vec_item(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { int optionals = v.buf->optionals; if (item == Py_None && (optionals & 1)) { @@ -49,14 +50,14 @@ static inline int unbox_vec_item(VecTExt v, PyObject *item, VecbufTExtItem *unbo unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; Py_INCREF(unboxed->buf); - return 1; + return 0; } } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { VecI64Object *o = (VecI64Object *)item; unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; Py_INCREF(unboxed->buf); - return 1; + return 0; } } else if (item->ob_type == &VecTExtType) { VecTExtObject *o = (VecTExtObject *)item; @@ -66,12 +67,12 @@ static inline int unbox_vec_item(VecTExt v, PyObject *item, VecbufTExtItem *unbo unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; Py_INCREF(unboxed->buf); - return 1; + return 0; } } // TODO: better error message PyErr_SetString(PyExc_TypeError, "invalid item type"); - return 0; + return -1; } // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. @@ -187,16 +188,13 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { } } -#if 0 - int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - // TODO VecTExt v = ((VecTExtObject *)self)->vec; - if (!VecTExt_ItemCheck(v, o)) - return -1; - if ((size_t)i < (size_t)v->len) { - Py_INCREF(o); - v->items[i] = o; + if ((size_t)i < (size_t)v.len) { + VecbufTExtItem item; + if (unbox_vec_item(v, o, &item) < 0) + return -1; + v.buf->items[i] = item; return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); @@ -204,6 +202,8 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } +#if 0 + PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { // TODO PyObject *res; From 457794b3a8bf76dd9c985fa65e60111a4b9d6bbd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 12:08:00 +0100 Subject: [PATCH 078/208] Comment out work-in-progress code (with missing implementations) --- mypyc/lib-rt/vecs/vec_t_ext.c | 26 ++++++++++++++++---------- mypyc/lib-rt/vecs/vecs.h | 7 ++++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 7c2b2cb89d3fe..fe406097a1c85 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -202,10 +202,11 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -#if 0 - PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { // TODO + return NULL; + + /* PyObject *res; if (op == Py_EQ || op == Py_NE) { if (other->ob_type != &VecTExtType) { @@ -224,37 +225,46 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { res = Py_NotImplemented; Py_INCREF(res); return res; + */ } -static int Vec_T_Ext_Remove(PyObject *self, PyObject *arg) { +static VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { // TODO + return Vec_T_Ext_Error(); + /* VecTExtObject *v = (VecTExtObject *)self; return vec_generic_remove(&v->len, v->items, arg); + */ } static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { // TODO + return NULL; + /* VecTExtObject *v = (VecTExtObject *)self; if (!VecTExt_ItemCheck(v, arg)) return NULL; if (!vec_generic_remove(&v->len, v->items, arg)) return NULL; Py_RETURN_NONE; + */ } -static VecTExt Vec_T_Ext_Pop(VecTExt vec, Py_ssize_t index, PyObject **result) { +static VecTExt Vec_T_Ext_Pop(VecTExt vec, Py_ssize_t index, VecbufTExtItem *result) { // TODO return Vec_T_Ext_Error(); } static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { // TODO + return NULL; + + /* VecTExtObject *v = (VecTExtObject *)self; return vec_generic_pop_wrapper(&v->len, v->items, args); + */ } -#endif - static int VecTExt_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -436,8 +446,6 @@ VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { } } -#if 0 - VecTExtFeatures TExtFeatures = { &VecTExtType, &VecbufTExtType, @@ -448,5 +456,3 @@ VecTExtFeatures TExtFeatures = { Vec_T_Ext_Remove, Vec_T_Ext_Slice, }; - -#endif diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index dec3cc1abc063..ef22900dfc4aa 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -132,9 +132,10 @@ typedef struct _VecTExtFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecTExt (*alloc)(Py_ssize_t, size_t, int optionals, int depth); - VecTExt (*append)(VecTExt, PyObject *); - VecTExt (*pop)(VecTExt, Py_ssize_t, PyObject **result); - VecTExt (*remove)(VecTExt, PyObject *); + PyObject *(*box)(VecTExt); + VecTExt (*append)(VecTExt, VecbufTExtItem); + VecTExt (*pop)(VecTExt, Py_ssize_t, VecbufTExtItem *result); + VecTExt (*remove)(VecTExt, VecbufTExtItem); // TODO: Py_ssize_t VecTExt (*slice)(VecTExt, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); From eb2c4aeb1bbfa4b07c35126177a62a247788399d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 12:13:08 +0100 Subject: [PATCH 079/208] Implement constructing empty nested vec --- mypyc/lib-rt/vecs/vecs.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 1e8b63d094d25..9573b264b12af 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -35,18 +35,16 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } else { return Vec_T_FromIterable(p->item_type, init); } -#if 0 } else { if (init == NULL) { - return (PyObject *)Vec_T_Ext_New(0, (PyObject *)p->item_type, p->optionals, - p->depth); + VecTExt vec = Vec_T_Ext_New(0, p->item_type, p->optionals, p->depth); + if (VEC_IS_ERROR(vec)) + return NULL; + return Vec_T_Ext_Box(vec); } else { - return (PyObject *)Vec_T_Ext_FromIterable((PyTypeObject *)p->item_type, p->optionals, - p->depth, init); + return Vec_T_Ext_FromIterable(p->item_type, p->optionals, p->depth, init); } -#endif } - return NULL; // TODO remove } static int From 936117fa8bb79b85bc4e72f631365d55c568a334 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 13:34:24 +0100 Subject: [PATCH 080/208] Implement append for nested vecs --- mypyc/lib-rt/vecs/vec_t_ext.c | 48 +++-------------------------------- mypyc/lib-rt/vecs/vecs.c | 16 +++++++----- mypyc/lib-rt/vecs/vecs.h | 42 ++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 52 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index fe406097a1c85..a537855d3d4ac 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -33,48 +33,6 @@ static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { } } -// Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. -// Return a new reference. -static inline int unbox_vec_item(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { - int optionals = v.buf->optionals; - if (item == Py_None && (optionals & 1)) { - unboxed->len = -1; - return 0; - } - int depth = v.buf->depth; - if (depth == 1) { - // TODO: vec[i64] - if (item->ob_type == &VecTType) { - VecTExtObject *o = (VecTExtObject *)item; - if (o->vec.buf->item_type == v.buf->item_type) { - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); - return 0; - } - } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { - VecI64Object *o = (VecI64Object *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); - return 0; - } - } else if (item->ob_type == &VecTExtType) { - VecTExtObject *o = (VecTExtObject *)item; - if (o->vec.buf->depth == v.buf->depth - 1 - && o->vec.buf->item_type == v.buf->item_type - && o->vec.buf->optionals == (optionals >> 1)) { - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); - return 0; - } - } - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); - return -1; -} - // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { @@ -192,7 +150,7 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTExt v = ((VecTExtObject *)self)->vec; if ((size_t)i < (size_t)v.len) { VecbufTExtItem item; - if (unbox_vec_item(v, o, &item) < 0) + if (Vec_T_Ext_UnboxItem(v, o, &item) < 0) return -1; v.buf->items[i] = item; return 0; @@ -397,7 +355,7 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { VecbufTExtItem vecitem; - if (unbox_vec_item(v, item, &vecitem) < 0) { + if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) { Py_DECREF(iter); VEC_DECREF(v); Py_DECREF(item); @@ -419,9 +377,9 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de return Vec_T_Ext_Box(v); } +// Steals references to vec and x VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { Py_ssize_t cap = VEC_CAP(vec); - Py_INCREF(x.buf); if (vec.len < cap) { vec.buf->items[vec.len] = x; vec.len++; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 9573b264b12af..def5d99b0828d 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -335,7 +335,7 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = Vec_I64_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; + return NULL; // TODO: decref? return Vec_I64_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; @@ -347,14 +347,16 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return Vec_T_Box(v); -#if 0 } else if (VecTExt_Check(vec)) { - VecTExtObject *v = (VecTExtObject *)vec; - if (!VecTExt_ItemCheck(v, item)) + VecTExt v = ((VecTExtObject *)vec)->vec; + VecbufTExtItem vecitem; + if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) return NULL; - Py_INCREF(vec); - return Vec_T_Ext_Append(vec, item); -#endif + VEC_INCREF(v); + v = Vec_T_Ext_Append(v, vecitem); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return Vec_T_Ext_Box(v); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index ef22900dfc4aa..82fd801943565 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -250,6 +250,48 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); +// Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. +// Return a new reference. +static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { + int optionals = v.buf->optionals; + if (item == Py_None && (optionals & 1)) { + unboxed->len = -1; + return 0; + } + int depth = v.buf->depth; + if (depth == 1) { + // TODO: vec[i64] + if (item->ob_type == &VecTType) { + VecTExtObject *o = (VecTExtObject *)item; + if (o->vec.buf->item_type == v.buf->item_type) { + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 0; + } + } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { + VecI64Object *o = (VecI64Object *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 0; + } + } else if (item->ob_type == &VecTExtType) { + VecTExtObject *o = (VecTExtObject *)item; + if (o->vec.buf->depth == v.buf->depth - 1 + && o->vec.buf->item_type == v.buf->item_type + && o->vec.buf->optionals == (optionals >> 1)) { + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + Py_INCREF(unboxed->buf); + return 0; + } + } + // TODO: better error message + PyErr_SetString(PyExc_TypeError, "invalid item type"); + return -1; +} + // Misc helpers static inline int check_float_error(PyObject *o) { From 8d192ae8d953db88f552bcafa2b122fd93482c1a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 16 Apr 2023 13:51:39 +0100 Subject: [PATCH 081/208] Some fixes --- mypyc/lib-rt/vecs/vecs.c | 5 +++-- mypyc/lib-rt/vecs/vecs.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index def5d99b0828d..215e45fb27fde 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -171,7 +171,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (p == NULL) return NULL; Py_INCREF(item); - p->item_type = (size_t)item | (optionals & 1); + p->item_type = (size_t)item | ((optionals >> depth) & 1); p->depth = depth; p->optionals = optionals; PyObject_GC_Track(p); @@ -196,6 +196,7 @@ PyTypeObject VecGenericType = { PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optionals, int verbose) { // TODO: Check for errors + // TODO: Refcount handling PyObject *l = Py_BuildValue("[]"); PyObject *prefix; PyObject *mid; @@ -219,7 +220,7 @@ PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optio for (Py_ssize_t i = 0; i < len; i++) { PyObject *it = PySequence_GetItem(vec, i); PyObject *r; - if (depth == 0) { + if (depth == 0 || it == Py_None) { r = PyObject_Repr(it); } else { r = vec_repr(it, item_type, depth - 1, optionals >> 1, 0); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 82fd801943565..17a7e793181cc 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -256,6 +256,7 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem int optionals = v.buf->optionals; if (item == Py_None && (optionals & 1)) { unboxed->len = -1; + unboxed->buf = NULL; return 0; } int depth = v.buf->depth; From 3d442264e84a47947a8fde113eb9228d4e15570e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 29 Apr 2023 13:54:06 +0100 Subject: [PATCH 082/208] Implement nested vec equality --- mypyc/lib-rt/vecs/vec_t_ext.c | 45 +++++++++++++++++++++++++---------- mypyc/lib-rt/vecs/vecs.h | 2 +- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index a537855d3d4ac..40adf8be39328 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -160,30 +160,49 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { - // TODO - return NULL; +static PyObject *compare_vec_t_ext_eq(VecTExt x, VecTExt y, int op) { + int cmp = 1; + PyObject *res; + if (x.len != y.len + || x.buf->item_type != y.buf->item_type + || x.buf->depth != y.buf->depth + || x.buf->optionals != y.buf->optionals) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < x.len; i++) { + PyObject *x_item = box_vec_item(x, i); + PyObject *y_item = box_vec_item(y, i); + int itemcmp = PyObject_RichCompareBool(x_item, y_item, Py_EQ); + Py_DECREF(x_item); + Py_DECREF(y_item); + if (itemcmp < 0) + return NULL; + if (!itemcmp) { + cmp = 0; + break; + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + Py_INCREF(res); + return res; +} - /* +PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { PyObject *res; if (op == Py_EQ || op == Py_NE) { if (other->ob_type != &VecTExtType) { res = op == Py_EQ ? Py_False : Py_True; } else { - VecTExtObject *x = (VecTExtObject *)self; - VecTExtObject *y = (VecTExtObject *)other; - if (x->item_type != y->item_type - || x->depth != y->depth - || x->optionals != y->optionals) { - res = op == Py_EQ ? Py_False : Py_True; - } else - return vec_generic_richcompare(&x->len, x->items, &y->len, y->items, op); + return compare_vec_t_ext_eq(((VecTExtObject *)self)->vec, + ((VecTExtObject *)other)->vec, op); } } else res = Py_NotImplemented; Py_INCREF(res); return res; - */ } static VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 17a7e793181cc..cf1382fa1ea96 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -31,7 +31,7 @@ typedef struct _VecbufTExtItem { PyObject *buf; } VecbufTExtItem; -// Extended generic vec type: vec[t | None], vec[vec[...]], etc. +// Nested vec type: vec[vec[...]], vec[vec[...] | None], etc. typedef struct _VecbufTExtObject { PyObject_VAR_HEAD // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. From 0a707d75dab8501cbde2b101c9be5d911563dafe Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 29 Apr 2023 18:43:22 +0100 Subject: [PATCH 083/208] Support remove with nested vec --- mypyc/lib-rt/vecs/vec_t_ext.c | 75 ++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 40adf8be39328..c7ddd3bf2e51b 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,8 +9,7 @@ #include #include "vecs.h" -static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { - VecbufTExtItem item = v.buf->items[index]; +static inline PyObject *box_vec_item(VecTExt v, VecbufTExtItem item) { if (item.len < 0) Py_RETURN_NONE; Py_INCREF(item.buf); @@ -33,6 +32,11 @@ static inline PyObject *box_vec_item(VecTExt v, Py_ssize_t index) { } } +static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { + return box_vec_item(v, v.buf->items[index]); +} + + // Alloc a partially initialized vec. Caller *must* initialize len and buf->items. static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { @@ -77,7 +81,7 @@ PyObject *vec_t_ext_repr(PyObject *self) { PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { VecTExt v = ((VecTExtObject *)o)->vec; if ((size_t)i < (size_t)v.len) { - return box_vec_item(v, i); + return box_vec_item_by_index(v, i); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -116,7 +120,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { if (i == -1 && PyErr_Occurred()) return NULL; if ((size_t)i < (size_t)vec.len) { - return box_vec_item(vec, i); + return box_vec_item_by_index(vec, i); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -170,8 +174,8 @@ static PyObject *compare_vec_t_ext_eq(VecTExt x, VecTExt y, int op) { cmp = 0; } else { for (Py_ssize_t i = 0; i < x.len; i++) { - PyObject *x_item = box_vec_item(x, i); - PyObject *y_item = box_vec_item(y, i); + PyObject *x_item = box_vec_item_by_index(x, i); + PyObject *y_item = box_vec_item_by_index(y, i); int itemcmp = PyObject_RichCompareBool(x_item, y_item, Py_EQ); Py_DECREF(x_item); Py_DECREF(y_item); @@ -206,25 +210,58 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { } static VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { - // TODO + VecbufTExtItem *items = self.buf->items; + + PyObject *boxed_arg = box_vec_item(self, arg); + if (boxed_arg == NULL) + return Vec_T_Ext_Error(); + + for (Py_ssize_t i = 0; i < self.len; i++) { + int match = 0; + if (items[i].len == arg.len && items[i].buf == arg.buf) + match = 1; + else { + PyObject *item = box_vec_item_by_index(self, i); + if (item == NULL) { + Py_DECREF(boxed_arg); + return Vec_T_Ext_Error(); + } + int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); + Py_DECREF(item); + if (itemcmp < 0) { + Py_DECREF(boxed_arg); + return Vec_T_Ext_Error(); + } + match = itemcmp; + } + if (match) { + Py_CLEAR(items[i].buf); + for (; i < self.len - 1; i++) { + items[i] = items[i + 1]; + } + if (self.len > 0) + Py_XINCREF(items[self.len - 1].buf); + self.len--; + Py_INCREF(self.buf); + Py_DECREF(boxed_arg); + return self; + } + } + Py_DECREF(boxed_arg); + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); return Vec_T_Ext_Error(); - /* - VecTExtObject *v = (VecTExtObject *)self; - return vec_generic_remove(&v->len, v->items, arg); - */ } static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { - // TODO - return NULL; - /* - VecTExtObject *v = (VecTExtObject *)self; - if (!VecTExt_ItemCheck(v, arg)) + VecTExt v = ((VecTExtObject *)self)->vec; + VecbufTExtItem item; + if (Vec_T_Ext_UnboxItem(v, arg, &item) < 0) return NULL; - if (!vec_generic_remove(&v->len, v->items, arg)) + v = Vec_T_Ext_Remove(v, item); + Py_DECREF(item.buf); + if (VEC_IS_ERROR(v)) return NULL; - Py_RETURN_NONE; - */ + return Vec_T_Ext_Box(v); } static VecTExt Vec_T_Ext_Pop(VecTExt vec, Py_ssize_t index, VecbufTExtItem *result) { From 7cd1ee92d08b2e8c51ef12245fd8b4355ca52952 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 29 Apr 2023 21:11:18 +0100 Subject: [PATCH 084/208] Support pop() with nested vecs --- mypyc/lib-rt/vecs/vec_t_ext.c | 57 +++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 9 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index c7ddd3bf2e51b..25fd9fc95c336 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -264,19 +264,58 @@ static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { return Vec_T_Ext_Box(v); } -static VecTExt Vec_T_Ext_Pop(VecTExt vec, Py_ssize_t index, VecbufTExtItem *result) { - // TODO - return Vec_T_Ext_Error(); +static VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result) { + if (index < 0) + index += v.len; + + if (index < 0 || index >= v.len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return Vec_T_Ext_Error(); + } + + VecbufTExtItem *items = v.buf->items; + *result = items[index]; + for (Py_ssize_t i = index; i < v.len - 1; i++) + items[i] = items[i + 1]; + if (v.len > 0) + Py_XINCREF(items[v.len - 1].buf); + v.len--; + VEC_INCREF(v); + return v; } static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { - // TODO - return NULL; + Py_ssize_t index = -1; + if (!PyArg_ParseTuple(args, "|n:pop", &index)) + return NULL; - /* - VecTExtObject *v = (VecTExtObject *)self; - return vec_generic_pop_wrapper(&v->len, v->items, args); - */ + VecTExt v = ((VecTExtObject *)self)->vec; + VecbufTExtItem result; + v = Vec_T_Ext_Pop(v, index, &result); + if (VEC_IS_ERROR(v)) + return NULL; + + PyObject *vboxed = Vec_T_Ext_Box(v); + if (vboxed == NULL) + return NULL; + + PyObject *result_boxed = box_vec_item(v, result); + if (result_boxed == NULL) { + Py_DECREF(vboxed); + Py_DECREF(result.buf); + } + + PyObject *res = PyTuple_New(2); + if (res == NULL) { + Py_DECREF(vboxed); + Py_DECREF(result.buf); + Py_DECREF(result_boxed); + return NULL; + } + + PyTuple_SetItem(res, 0, vboxed); + PyTuple_SetItem(res, 1, result_boxed); + return res; } static int From a615afed4ecfbca04d404ba4057c3f55cc9cdd98 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 May 2023 14:46:39 +0100 Subject: [PATCH 085/208] Add vec[i64] unbox to public API --- mypyc/lib-rt/vecs/vec_i64.c | 13 +++++++++++++ mypyc/lib-rt/vecs/vec_t.c | 1 + mypyc/lib-rt/vecs/vec_t_ext.c | 1 + mypyc/lib-rt/vecs/vecs.h | 3 +++ 4 files changed, 18 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index d7ac79d1fe863..3efb422dbfedb 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -31,6 +31,18 @@ PyObject *Vec_I64_Box(VecI64 vec) { return (PyObject *)obj; } +VecI64 Vec_I64_Unbox(PyObject *obj) { + if (obj->ob_type == &VecI64Type) { + VecI64 result = ((VecI64Object *)obj)->vec; + VEC_INCREF(result); // TODO: Should we borrow instead? + return result; + } else { + // TODO: Better error message + PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); + return Vec_I64_Error(); + } +} + VecI64 Vec_I64_New(Py_ssize_t size) { VecI64 vec = vec_i64_alloc(size); if (VEC_IS_ERROR(vec)) @@ -357,6 +369,7 @@ VecI64Features I64Features = { &VecbufI64Type, Vec_I64_New, Vec_I64_Box, + Vec_I64_Unbox, Vec_I64_Append, Vec_I64_Pop, Vec_I64_Remove, diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 74e354a6d3b37..2dea27dd4ee48 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -423,6 +423,7 @@ VecTFeatures TFeatures = { &VecbufTType, Vec_T_New, Vec_T_Box, + NULL, // TODO: unbox Vec_T_Append, Vec_T_Pop, Vec_T_Remove, diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 25fd9fc95c336..3b1ec45838376 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -504,6 +504,7 @@ VecTExtFeatures TExtFeatures = { &VecbufTExtType, Vec_T_Ext_New, Vec_T_Ext_Box, + NULL, // TODO: unbox Vec_T_Ext_Append, Vec_T_Ext_Pop, Vec_T_Ext_Remove, diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index cf1382fa1ea96..a978cb94c6c21 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -95,6 +95,7 @@ typedef struct _VecI64Features { PyTypeObject *buf_type; VecI64 (*alloc)(Py_ssize_t); PyObject *(*box)(VecI64); + VecI64 (*unbox)(PyObject *); VecI64 (*append)(VecI64, int64_t); VecI64 (*pop)(VecI64, Py_ssize_t, int64_t *result); VecI64 (*remove)(VecI64, int64_t); @@ -114,6 +115,7 @@ typedef struct _VecTFeatures { PyTypeObject *buf_type; VecT (*alloc)(Py_ssize_t, size_t); PyObject *(*box)(VecT); + VecT (*unbox)(size_t); VecT (*append)(VecT, PyObject *); VecT (*pop)(VecT, Py_ssize_t, PyObject **result); VecT (*remove)(VecT, PyObject *); @@ -133,6 +135,7 @@ typedef struct _VecTExtFeatures { PyTypeObject *buf_type; VecTExt (*alloc)(Py_ssize_t, size_t, int optionals, int depth); PyObject *(*box)(VecTExt); + VecTExt (*unbox)(size_t, int optionals, int depth); VecTExt (*append)(VecTExt, VecbufTExtItem); VecTExt (*pop)(VecTExt, Py_ssize_t, VecbufTExtItem *result); VecTExt (*remove)(VecTExt, VecbufTExtItem); From 29d7060e799299eab06aa88990ac3ab55bef1505 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 11:50:02 +0100 Subject: [PATCH 086/208] Make remove() a module-level function --- mypyc/lib-rt/vecs/vec_i64.c | 17 +------------ mypyc/lib-rt/vecs/vec_t.c | 13 +--------- mypyc/lib-rt/vecs/vec_t_ext.c | 15 +---------- mypyc/lib-rt/vecs/vecs.c | 48 +++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 3 +++ 5 files changed, 54 insertions(+), 42 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 3efb422dbfedb..05732f7c749d9 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -221,7 +221,7 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { +VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -236,20 +236,6 @@ static VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { return Vec_I64_Error(); } -static PyObject *vec_i64_remove(PyObject *self, PyObject *arg) { - if (check_float_error(arg)) - return NULL; - long long x = PyLong_AsLongLong(arg); - if (x == -1 && PyErr_Occurred()) - return NULL; - - VecI64Object *vec = (VecI64Object *)self; - VecI64 res = Vec_I64_Remove(vec->vec, x); - if (VEC_IS_ERROR(res)) - return NULL; - return Vec_I64_Box(res); -} - static VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { if (index < 0) index += v.len; @@ -311,7 +297,6 @@ static PySequenceMethods VecI64Sequence = { }; static PyMethodDef vec_i64_methods[] = { - {"remove", vec_i64_remove, METH_O, NULL}, {"pop", vec_i64_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 2dea27dd4ee48..198bddedf9472 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -156,7 +156,7 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static VecT Vec_T_Remove(VecT v, PyObject *arg) { +VecT Vec_T_Remove(VecT v, PyObject *arg) { PyObject **items = v.buf->items; for (Py_ssize_t i = 0; i < v.len; i++) { int match = 0; @@ -184,16 +184,6 @@ static VecT Vec_T_Remove(VecT v, PyObject *arg) { return Vec_T_Error(); } -static PyObject *vec_t_remove(PyObject *self, PyObject *arg) { - VecT v = ((VecTObject *)self)->vec; - if (!VecT_ItemCheck(v, arg)) - return NULL; - v = Vec_T_Remove(v, arg); - if (VEC_IS_ERROR(v)) - return NULL; - return Vec_T_Box(v); -} - static VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { if (index < 0) index += v.len; @@ -320,7 +310,6 @@ static PySequenceMethods VecTSequence = { }; static PyMethodDef vec_t_methods[] = { - {"remove", vec_t_remove, METH_O, NULL}, {"pop", vec_t_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 3b1ec45838376..d48ecbca5fc63 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -209,7 +209,7 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { return res; } -static VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { +VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; PyObject *boxed_arg = box_vec_item(self, arg); @@ -252,18 +252,6 @@ static VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { return Vec_T_Ext_Error(); } -static PyObject *vec_t_ext_remove(PyObject *self, PyObject *arg) { - VecTExt v = ((VecTExtObject *)self)->vec; - VecbufTExtItem item; - if (Vec_T_Ext_UnboxItem(v, arg, &item) < 0) - return NULL; - v = Vec_T_Ext_Remove(v, item); - Py_DECREF(item.buf); - if (VEC_IS_ERROR(v)) - return NULL; - return Vec_T_Ext_Box(v); -} - static VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result) { if (index < 0) index += v.len; @@ -397,7 +385,6 @@ static PySequenceMethods VecTExtSequence = { }; static PyMethodDef vec_t_ext_methods[] = { - {"remove", vec_t_ext_remove, METH_O, NULL}, {"pop", vec_t_ext_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 215e45fb27fde..8ab4b0c995512 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -364,8 +364,56 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) } } +static PyObject *vecs_remove(PyObject *self, PyObject *args) +{ + PyObject *vec; + PyObject *item; + + if (!PyArg_ParseTuple(args, "OO", &vec, &item)) + return NULL; + + if (VecI64_Check(vec)) { + if (check_float_error(item)) + return NULL; + int64_t x = PyLong_AsLong(item); + if (x == -1 && PyErr_Occurred()) { + return NULL; + } + VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); + v = Vec_I64_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return Vec_I64_Box(v); + } else if (VecT_Check(vec)) { + VecT v = ((VecTObject *)vec)->vec; + if (!VecT_ItemCheck(v, item)) { + return NULL; + } + VEC_INCREF(v); + v = Vec_T_Remove(v, item); + if (VEC_IS_ERROR(v)) + return NULL; + return Vec_T_Box(v); + } else if (VecTExt_Check(vec)) { + VecTExt v = ((VecTExtObject *)vec)->vec; + VecbufTExtItem vecitem; + if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) + return NULL; + VEC_INCREF(v); + v = Vec_T_Ext_Remove(v, vecitem); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return Vec_T_Ext_Box(v); + } else { + PyErr_SetString(PyExc_TypeError, "vec argument expected"); + return NULL; + } +} + static PyMethodDef VecsMethods[] = { {"append", vecs_append, METH_VARARGS, "Append a value to a vec"}, + {"remove", vecs_remove, METH_VARARGS, "Remove a value from a vec"}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index a978cb94c6c21..bc92a3e2b488c 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -198,6 +198,7 @@ static inline int VecI64_Check(PyObject *o) { PyObject *Vec_I64_Box(VecI64); VecI64 Vec_I64_Append(VecI64, int64_t x); +VecI64 Vec_I64_Remove(VecI64, int64_t x); // vec[t] operations (simple) @@ -221,6 +222,7 @@ VecT Vec_T_New(Py_ssize_t size, size_t item_type); PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT); VecT Vec_T_Append(VecT vec, PyObject *x); +VecT Vec_T_Remove(VecT vec, PyObject *x); // vec[t] operations (extended) @@ -252,6 +254,7 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de PyObject *iterable); PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); +VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a new reference. From 60fbfef314a72a4ace8397a8af6f6703ef929296 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 15:37:18 +0100 Subject: [PATCH 087/208] Make pop() a module-level function --- mypyc/lib-rt/vecs/vec_i64.c | 38 ++--------------- mypyc/lib-rt/vecs/vec_t.c | 30 +------------ mypyc/lib-rt/vecs/vec_t_ext.c | 64 ++-------------------------- mypyc/lib-rt/vecs/vecs.c | 79 ++++++++++++++++++++++++++++++++++- mypyc/lib-rt/vecs/vecs.h | 26 ++++++++++++ 5 files changed, 110 insertions(+), 127 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 05732f7c749d9..9f5f07fc51f94 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -236,7 +236,7 @@ VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { return Vec_I64_Error(); } -static VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { +VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { if (index < 0) index += v.len; @@ -246,46 +246,15 @@ static VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { } *result = v.buf->items[index]; - for (Py_ssize_t i = index; i < v.len - 1; i++) + for (Py_ssize_t i = index; i < v.len - 1; i++) { v.buf->items[i] = v.buf->items[i + 1]; + } v.len--; VEC_INCREF(v); return v; } -static PyObject *vec_i64_pop(PyObject *self, PyObject *args) { - Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:pop", &index)) - return NULL; - - int64_t x; - VecI64Object *vec = (VecI64Object *)self; - VecI64 vec2 = Vec_I64_Pop(vec->vec, index, &x); - if (VEC_IS_ERROR(vec2)) - return NULL; - - PyObject *item0 = Vec_I64_Box(vec2); - if (item0 == NULL) - return NULL; - PyObject *item1 = PyLong_FromLongLong(x); - if (item1 == NULL) { - Py_DECREF(item0); - return NULL; - } - - PyObject *res = PyTuple_New(2); - if (res == NULL) { - Py_DECREF(item0); - Py_DECREF(item1); - return NULL; - } - - PyTuple_SetItem(res, 0, item0); - PyTuple_SetItem(res, 1, item1); - return res; -} - static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, .mp_subscript = vec_i64_subscript, @@ -297,7 +266,6 @@ static PySequenceMethods VecI64Sequence = { }; static PyMethodDef vec_i64_methods[] = { - {"pop", vec_i64_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 198bddedf9472..9b1a2162cc543 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -184,7 +184,7 @@ VecT Vec_T_Remove(VecT v, PyObject *arg) { return Vec_T_Error(); } -static VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { +VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { if (index < 0) index += v.len; @@ -204,33 +204,6 @@ static VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { return v; } -static PyObject *vec_t_pop(PyObject *self, PyObject *args) { - Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:pop", &index)) - return NULL; - - VecT v = ((VecTObject *)self)->vec; - PyObject *result; - v = Vec_T_Pop(v, index, &result); - if (VEC_IS_ERROR(v)) - return NULL; - - PyObject *vboxed = Vec_T_Box(v); - if (vboxed == NULL) - return NULL; - - PyObject *res = PyTuple_New(2); - if (res == NULL) { - Py_DECREF(vboxed); - Py_DECREF(result); - return NULL; - } - - PyTuple_SetItem(res, 0, vboxed); - PyTuple_SetItem(res, 1, result); - return res; -} - static int VecT_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -310,7 +283,6 @@ static PySequenceMethods VecTSequence = { }; static PyMethodDef vec_t_methods[] = { - {"pop", vec_t_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index d48ecbca5fc63..fd77b9ba8fe03 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,31 +9,8 @@ #include #include "vecs.h" -static inline PyObject *box_vec_item(VecTExt v, VecbufTExtItem item) { - if (item.len < 0) - Py_RETURN_NONE; - Py_INCREF(item.buf); - if (v.buf->depth > 1) { - // Item is a nested vec - VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; - return Vec_T_Ext_Box(v); - } else { - // Item is a non-nested vec - void *item_type = (void *)(v.buf->item_type & ~1); - if (item_type == I64TypeObj) { - // vec[i64] - VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; - return Vec_I64_Box(v); - } else { - // Generic vec[t] - VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; - return Vec_T_Box(v); - } - } -} - static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { - return box_vec_item(v, v.buf->items[index]); + return Vec_T_Ext_BoxItem(v, v.buf->items[index]); } @@ -212,7 +189,7 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; - PyObject *boxed_arg = box_vec_item(self, arg); + PyObject *boxed_arg = Vec_T_Ext_BoxItem(self, arg); if (boxed_arg == NULL) return Vec_T_Ext_Error(); @@ -252,7 +229,7 @@ VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { return Vec_T_Ext_Error(); } -static VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result) { +VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result) { if (index < 0) index += v.len; @@ -272,40 +249,6 @@ static VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result return v; } -static PyObject *vec_t_ext_pop(PyObject *self, PyObject *args) { - Py_ssize_t index = -1; - if (!PyArg_ParseTuple(args, "|n:pop", &index)) - return NULL; - - VecTExt v = ((VecTExtObject *)self)->vec; - VecbufTExtItem result; - v = Vec_T_Ext_Pop(v, index, &result); - if (VEC_IS_ERROR(v)) - return NULL; - - PyObject *vboxed = Vec_T_Ext_Box(v); - if (vboxed == NULL) - return NULL; - - PyObject *result_boxed = box_vec_item(v, result); - if (result_boxed == NULL) { - Py_DECREF(vboxed); - Py_DECREF(result.buf); - } - - PyObject *res = PyTuple_New(2); - if (res == NULL) { - Py_DECREF(vboxed); - Py_DECREF(result.buf); - Py_DECREF(result_boxed); - return NULL; - } - - PyTuple_SetItem(res, 0, vboxed); - PyTuple_SetItem(res, 1, result_boxed); - return res; -} - static int VecTExt_traverse(VecTObject *self, visitproc visit, void *arg) { @@ -385,7 +328,6 @@ static PySequenceMethods VecTExtSequence = { }; static PyMethodDef vec_t_ext_methods[] = { - {"pop", vec_t_ext_pop, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 8ab4b0c995512..5e9e1dabb2e65 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -411,9 +411,84 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) } } +static PyObject *vecs_pop(PyObject *self, PyObject *args) +{ + PyObject *vec; + Py_ssize_t index = -1; + + if (!PyArg_ParseTuple(args, "O|n:pop", &vec, &index)) + return NULL; + + PyObject *result_item0; + PyObject *result_item1; + + if (VecI64_Check(vec)) { + VecI64 v = ((VecI64Object *)vec)->vec; + int64_t item; + v = Vec_I64_Pop(v, index, &item); + if (VEC_IS_ERROR(v)) + return NULL; + + result_item0 = Vec_I64_Box(v); + if (result_item0 == NULL) + return NULL; + result_item1 = PyLong_FromLongLong(item); + } else if (VecT_Check(vec)) { + VecT v = ((VecTObject *)vec)->vec; + v = Vec_T_Pop(v, index, &result_item1); + if (VEC_IS_ERROR(v)) + return NULL; + + result_item0 = Vec_T_Box(v); + if (result_item0 == NULL) { + Py_DECREF(result_item1); + return NULL; + } + } else if (VecTExt_Check(vec)) { + VecTExt v = ((VecTExtObject *)vec)->vec; + VecbufTExtItem item; + v = Vec_T_Ext_Pop(v, index, &item); + if (VEC_IS_ERROR(v)) + return NULL; + + result_item0 = Vec_T_Ext_Box(v); + if (result_item0 == NULL) { + Py_DECREF(item.buf); + return NULL; + } + + result_item1 = Vec_T_Ext_BoxItem(v, item); + if (result_item1 == NULL) { + Py_DECREF(result_item0); + Py_DECREF(item.buf); + return NULL; + } + } else { + PyErr_SetString(PyExc_TypeError, "vec argument expected"); + return NULL; + } + + if (result_item1 == NULL) { + Py_DECREF(result_item0); + return NULL; + } + + PyObject *res = PyTuple_New(2); + if (res == NULL) { + Py_DECREF(result_item0); + Py_DECREF(result_item1); + return NULL; + } + + PyTuple_SET_ITEM(res, 0, result_item0); + PyTuple_SET_ITEM(res, 1, result_item1); + return res; +} + static PyMethodDef VecsMethods[] = { - {"append", vecs_append, METH_VARARGS, "Append a value to a vec"}, - {"remove", vecs_remove, METH_VARARGS, "Remove a value from a vec"}, + {"append", vecs_append, METH_VARARGS, "Append a value to the end of a vec"}, + {"remove", vecs_remove, METH_VARARGS, "Remove first occurrence of value"}, + {"pop", vecs_pop, METH_VARARGS, "Remove and return item at index (default last)"}, {NULL, NULL, 0, NULL} /* Sentinel */ }; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index bc92a3e2b488c..bd6d163657ffc 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -199,6 +199,7 @@ static inline int VecI64_Check(PyObject *o) { PyObject *Vec_I64_Box(VecI64); VecI64 Vec_I64_Append(VecI64, int64_t x); VecI64 Vec_I64_Remove(VecI64, int64_t x); +VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result); // vec[t] operations (simple) @@ -223,6 +224,7 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT); VecT Vec_T_Append(VecT vec, PyObject *x); VecT Vec_T_Remove(VecT vec, PyObject *x); +VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result); // vec[t] operations (extended) @@ -255,6 +257,7 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); +VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a new reference. @@ -299,6 +302,29 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem return -1; } +static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { + if (item.len < 0) + Py_RETURN_NONE; + Py_INCREF(item.buf); + if (v.buf->depth > 1) { + // Item is a nested vec + VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; + return Vec_T_Ext_Box(v); + } else { + // Item is a non-nested vec + void *item_type = (void *)(v.buf->item_type & ~1); + if (item_type == I64TypeObj) { + // vec[i64] + VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; + return Vec_I64_Box(v); + } else { + // Generic vec[t] + VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; + return Vec_T_Box(v); + } + } +} + // Misc helpers static inline int check_float_error(PyObject *o) { From f8083d86ef1fe66cef929c26ba9c95f19e536632 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 8 May 2023 15:39:08 +0100 Subject: [PATCH 088/208] Refactoring --- mypyc/lib-rt/vecs/vec_i64.c | 40 +++++++++++++------------- mypyc/lib-rt/vecs/vec_t.c | 50 ++++++++++++++++---------------- mypyc/lib-rt/vecs/vec_t_ext.c | 54 +++++++++++++++++------------------ 3 files changed, 72 insertions(+), 72 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 9f5f07fc51f94..a9a91bfa14adf 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -221,6 +221,26 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } +VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { + if (vec.buf && vec.len < VEC_CAP(vec)) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; + } else { + Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; + Py_ssize_t new_size = 2 * cap + 1; + VecI64 new = vec_i64_alloc(new_size); + if (VEC_IS_ERROR(new)) + return Vec_I64_Error(); + new.len = vec.len + 1; + if (vec.len > 0) + memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); + new.buf->items[vec.len] = x; + Py_XDECREF(vec.buf); + return new; + } +} + VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { @@ -297,26 +317,6 @@ PyTypeObject VecI64Type = { .tp_methods = vec_i64_methods, }; -VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { - if (vec.buf && vec.len < VEC_CAP(vec)) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; - Py_ssize_t new_size = 2 * cap + 1; - VecI64 new = vec_i64_alloc(new_size); - if (VEC_IS_ERROR(new)) - return Vec_I64_Error(); - new.len = vec.len + 1; - if (vec.len > 0) - memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); - new.buf->items[vec.len] = x; - Py_XDECREF(vec.buf); - return new; - } -} - VecI64Features I64Features = { &VecI64Type, &VecbufI64Type, diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 9b1a2162cc543..29c2471740c0b 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -156,6 +156,31 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } +VecT Vec_T_Append(VecT vec, PyObject *x) { + Py_ssize_t cap = VEC_CAP(vec); + Py_INCREF(x); + if (vec.len < cap) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + // TODO: Avoid initializing to zero here + VecT new = vec_t_alloc(new_size, vec.buf->item_type); + if (VEC_IS_ERROR(new)) + return new; + // Copy items to new vec. + memcpy(new.buf->items, vec.buf->items, sizeof(PyObject *) * vec.len); + memset(new.buf->items + vec.len, 0, sizeof(PyObject *) * (new_size - vec.len)); + // Clear the items in the old vec. We avoid reference count manipulation. + memset(vec.buf->items, 0, sizeof(PyObject *) * vec.len); + new.buf->items[vec.len] = x; + new.len = vec.len + 1; + VEC_DECREF(vec); + return new; + } +} + VecT Vec_T_Remove(VecT v, PyObject *arg) { PyObject **items = v.buf->items; for (Py_ssize_t i = 0; i < v.len; i++) { @@ -354,31 +379,6 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { return Vec_T_Box(v); } -VecT Vec_T_Append(VecT vec, PyObject *x) { - Py_ssize_t cap = VEC_CAP(vec); - Py_INCREF(x); - if (vec.len < cap) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t new_size = 2 * cap + 1; - // TODO: Avoid initializing to zero here - VecT new = vec_t_alloc(new_size, vec.buf->item_type); - if (VEC_IS_ERROR(new)) - return new; - // Copy items to new vec. - memcpy(new.buf->items, vec.buf->items, sizeof(PyObject *) * vec.len); - memset(new.buf->items + vec.len, 0, sizeof(PyObject *) * (new_size - vec.len)); - // Clear the items in the old vec. We avoid reference count manipulation. - memset(vec.buf->items, 0, sizeof(PyObject *) * vec.len); - new.buf->items[vec.len] = x; - new.len = vec.len + 1; - VEC_DECREF(vec); - return new; - } -} - VecTFeatures TFeatures = { &VecTType, &VecbufTType, diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index fd77b9ba8fe03..9a8e2e69e54ac 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -186,6 +186,33 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { return res; } +// Steals references to vec and x +VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { + Py_ssize_t cap = VEC_CAP(vec); + if (vec.len < cap) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; + } else { + Py_ssize_t new_size = 2 * cap + 1; + // TODO: Avoid initializing to zero here + VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->optionals, + vec.buf->depth); + if (VEC_IS_ERROR(new)) + return new; + // Copy items to new vec. + memcpy(new.buf->items, vec.buf->items, sizeof(VecbufTExtItem) * vec.len); + // TODO: How to safely represent deleted items? + memset(new.buf->items + vec.len, 0, sizeof(VecbufTExtItem) * (new_size - vec.len)); + // Clear the items in the old vec. We avoid reference count manipulation. + memset(vec.buf->items, 0, sizeof(VecbufTExtItem) * vec.len); + new.buf->items[vec.len] = x; + new.len = vec.len + 1; + VEC_DECREF(vec); + return new; + } +} + VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; @@ -401,33 +428,6 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de return Vec_T_Ext_Box(v); } -// Steals references to vec and x -VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { - Py_ssize_t cap = VEC_CAP(vec); - if (vec.len < cap) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t new_size = 2 * cap + 1; - // TODO: Avoid initializing to zero here - VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->optionals, - vec.buf->depth); - if (VEC_IS_ERROR(new)) - return new; - // Copy items to new vec. - memcpy(new.buf->items, vec.buf->items, sizeof(VecbufTExtItem) * vec.len); - // TODO: How to safely represent deleted items? - memset(new.buf->items + vec.len, 0, sizeof(VecbufTExtItem) * (new_size - vec.len)); - // Clear the items in the old vec. We avoid reference count manipulation. - memset(vec.buf->items, 0, sizeof(VecbufTExtItem) * vec.len); - new.buf->items[vec.len] = x; - new.len = vec.len + 1; - VEC_DECREF(vec); - return new; - } -} - VecTExtFeatures TExtFeatures = { &VecTExtType, &VecbufTExtType, From 21bebe3833058cb43502d71de2c147fb7eba71f0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 11:56:13 +0100 Subject: [PATCH 089/208] Update native pop() implementation to return a tuple --- mypyc/lib-rt/vecs/vec_i64.c | 13 +++++++++---- mypyc/lib-rt/vecs/vec_t.c | 13 +++++++++---- mypyc/lib-rt/vecs/vec_t_ext.c | 14 ++++++++++---- mypyc/lib-rt/vecs/vecs.c | 35 +++++++++++++++++++---------------- mypyc/lib-rt/vecs/vecs.h | 27 +++++++++++++++++++++------ 5 files changed, 68 insertions(+), 34 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index a9a91bfa14adf..e5ff0c4968320 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -256,23 +256,28 @@ VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { return Vec_I64_Error(); } -VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result) { +VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index) { + VecI64PopResult result; + if (index < 0) index += v.len; if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - return Vec_I64_Error(); + result.vec = Vec_I64_Error(); + result.item = 0; + return result; } - *result = v.buf->items[index]; + result.item = v.buf->items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) { v.buf->items[i] = v.buf->items[i + 1]; } v.len--; VEC_INCREF(v); - return v; + result.vec = v; + return result; } static PyMappingMethods VecI64Mapping = { diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 29c2471740c0b..cb7e6f5bc9abd 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -209,24 +209,29 @@ VecT Vec_T_Remove(VecT v, PyObject *arg) { return Vec_T_Error(); } -VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result) { +VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index) { + VecTPopResult result; + if (index < 0) index += v.len; if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - return Vec_T_Error(); + result.vec = Vec_T_Error(); + result.item = NULL; + return result; } PyObject **items = v.buf->items; - *result = items[index]; + result.item = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; if (v.len > 0) Py_XINCREF(items[v.len - 1]); v.len--; VEC_INCREF(v); - return v; + result.vec = v; + return result; } static int diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 9a8e2e69e54ac..bd60b67c1cd4d 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -256,24 +256,30 @@ VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { return Vec_T_Ext_Error(); } -VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result) { +VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index) { + VecTExtPopResult result; + if (index < 0) index += v.len; if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - return Vec_T_Ext_Error(); + result.vec = Vec_T_Ext_Error(); + result.item.len = 0; + result.item.buf = NULL; + return result; } VecbufTExtItem *items = v.buf->items; - *result = items[index]; + result.item = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; if (v.len > 0) Py_XINCREF(items[v.len - 1].buf); v.len--; VEC_INCREF(v); - return v; + result.vec = v; + return result; } static int diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 5e9e1dabb2e65..f2d8214a4ed1e 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -424,43 +424,46 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) if (VecI64_Check(vec)) { VecI64 v = ((VecI64Object *)vec)->vec; - int64_t item; - v = Vec_I64_Pop(v, index, &item); - if (VEC_IS_ERROR(v)) + VecI64PopResult r; + r = Vec_I64_Pop(v, index); + if (VEC_IS_ERROR(r.vec)) return NULL; - result_item0 = Vec_I64_Box(v); + result_item0 = Vec_I64_Box(r.vec); if (result_item0 == NULL) return NULL; - result_item1 = PyLong_FromLongLong(item); + result_item1 = PyLong_FromLongLong(r.item); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; - v = Vec_T_Pop(v, index, &result_item1); - if (VEC_IS_ERROR(v)) + VecTPopResult r; + r = Vec_T_Pop(v, index); + if (VEC_IS_ERROR(r.vec)) return NULL; - result_item0 = Vec_T_Box(v); + result_item0 = Vec_T_Box(r.vec); if (result_item0 == NULL) { - Py_DECREF(result_item1); + Py_DECREF(r.item); return NULL; } + result_item1 = r.item; } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; - VecbufTExtItem item; - v = Vec_T_Ext_Pop(v, index, &item); - if (VEC_IS_ERROR(v)) + VecTExtPopResult r; + r = Vec_T_Ext_Pop(v, index); + if (VEC_IS_ERROR(r.vec)) return NULL; - result_item0 = Vec_T_Ext_Box(v); + result_item0 = Vec_T_Ext_Box(r.vec); if (result_item0 == NULL) { - Py_DECREF(item.buf); + Py_DECREF(r.vec.buf); + Py_DECREF(r.item.buf); return NULL; } - result_item1 = Vec_T_Ext_BoxItem(v, item); + result_item1 = Vec_T_Ext_BoxItem(r.vec, r.item); if (result_item1 == NULL) { Py_DECREF(result_item0); - Py_DECREF(item.buf); + Py_DECREF(r.item.buf); return NULL; } } else { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index bd6d163657ffc..6b6847bf8c33a 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -89,6 +89,11 @@ typedef struct _VecTExtObject { } VecTExtObject; +typedef struct { + VecI64 vec; + int64_t item; +} VecI64PopResult; + // vec[i64] operations + type objects (stored in a capsule) typedef struct _VecI64Features { PyTypeObject *boxed_type; @@ -97,7 +102,7 @@ typedef struct _VecI64Features { PyObject *(*box)(VecI64); VecI64 (*unbox)(PyObject *); VecI64 (*append)(VecI64, int64_t); - VecI64 (*pop)(VecI64, Py_ssize_t, int64_t *result); + VecI64PopResult (*pop)(VecI64, Py_ssize_t); VecI64 (*remove)(VecI64, int64_t); // TODO: Py_ssize_t VecI64 (*slice)(VecI64, int64_t, int64_t); @@ -107,6 +112,11 @@ typedef struct _VecI64Features { // iter? } VecI64Features; +typedef struct { + VecT vec; + PyObject *item; +} VecTPopResult; + // vec[T] operations + type object (stored in a capsule) // // T is a class type @@ -117,7 +127,7 @@ typedef struct _VecTFeatures { PyObject *(*box)(VecT); VecT (*unbox)(size_t); VecT (*append)(VecT, PyObject *); - VecT (*pop)(VecT, Py_ssize_t, PyObject **result); + VecTPopResult (*pop)(VecT, Py_ssize_t); VecT (*remove)(VecT, PyObject *); // TODO: Py_ssize_t VecT (*slice)(VecT, int64_t, int64_t); @@ -127,6 +137,11 @@ typedef struct _VecTFeatures { // iter? } VecTFeatures; +typedef struct { + VecTExt vec; + VecbufTExtItem item; +} VecTExtPopResult; + // vec[T] operations for complex item types + type object (stored in a capsule) // // T can be T | None or vec[T] (or a combination of these) @@ -137,7 +152,7 @@ typedef struct _VecTExtFeatures { PyObject *(*box)(VecTExt); VecTExt (*unbox)(size_t, int optionals, int depth); VecTExt (*append)(VecTExt, VecbufTExtItem); - VecTExt (*pop)(VecTExt, Py_ssize_t, VecbufTExtItem *result); + VecTExtPopResult (*pop)(VecTExt, Py_ssize_t); VecTExt (*remove)(VecTExt, VecbufTExtItem); // TODO: Py_ssize_t VecTExt (*slice)(VecTExt, int64_t, int64_t); @@ -199,7 +214,7 @@ static inline int VecI64_Check(PyObject *o) { PyObject *Vec_I64_Box(VecI64); VecI64 Vec_I64_Append(VecI64, int64_t x); VecI64 Vec_I64_Remove(VecI64, int64_t x); -VecI64 Vec_I64_Pop(VecI64 v, Py_ssize_t index, int64_t *result); +VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index); // vec[t] operations (simple) @@ -224,7 +239,7 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT); VecT Vec_T_Append(VecT vec, PyObject *x); VecT Vec_T_Remove(VecT vec, PyObject *x); -VecT Vec_T_Pop(VecT v, Py_ssize_t index, PyObject **result); +VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index); // vec[t] operations (extended) @@ -257,7 +272,7 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t de PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); -VecTExt Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index, VecbufTExtItem *result); +VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a new reference. From cae209cb69abbaf45ecac695ef454383b9d32266 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 29 May 2023 12:14:16 +0100 Subject: [PATCH 090/208] Use tuple return type that is compatible with mypyc --- mypyc/lib-rt/vecs/vec_i64.c | 8 ++++---- mypyc/lib-rt/vecs/vec_t.c | 8 ++++---- mypyc/lib-rt/vecs/vecs.c | 14 +++++++------- mypyc/lib-rt/vecs/vecs.h | 28 ++++++++++++++++++++-------- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index e5ff0c4968320..c9855bee27aa5 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -264,19 +264,19 @@ VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.vec = Vec_I64_Error(); - result.item = 0; + result.f0 = Vec_I64_Error(); + result.f1 = 0; return result; } - result.item = v.buf->items[index]; + result.f1 = v.buf->items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) { v.buf->items[i] = v.buf->items[i + 1]; } v.len--; VEC_INCREF(v); - result.vec = v; + result.f0 = v; return result; } diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index cb7e6f5bc9abd..9ad6d47f53704 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -217,20 +217,20 @@ VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.vec = Vec_T_Error(); - result.item = NULL; + result.f0 = Vec_T_Error(); + result.f1 = NULL; return result; } PyObject **items = v.buf->items; - result.item = items[index]; + result.f1 = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; if (v.len > 0) Py_XINCREF(items[v.len - 1]); v.len--; VEC_INCREF(v); - result.vec = v; + result.f0 = v; return result; } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index f2d8214a4ed1e..dc7df8bfa8428 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -426,26 +426,26 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) VecI64 v = ((VecI64Object *)vec)->vec; VecI64PopResult r; r = Vec_I64_Pop(v, index); - if (VEC_IS_ERROR(r.vec)) + if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_I64_Box(r.vec); + result_item0 = Vec_I64_Box(r.f0); if (result_item0 == NULL) return NULL; - result_item1 = PyLong_FromLongLong(r.item); + result_item1 = PyLong_FromLongLong(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; r = Vec_T_Pop(v, index); - if (VEC_IS_ERROR(r.vec)) + if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_T_Box(r.vec); + result_item0 = Vec_T_Box(r.f0); if (result_item0 == NULL) { - Py_DECREF(r.item); + Py_DECREF(r.f1); return NULL; } - result_item1 = r.item; + result_item1 = r.f1; } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecTExtPopResult r; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 6b6847bf8c33a..546a2be031070 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -89,10 +89,16 @@ typedef struct _VecTExtObject { } VecTExtObject; -typedef struct { - VecI64 vec; - int64_t item; -} VecI64PopResult; +#ifndef MYPYC_DECLARED_tuple_T2V88 +#define MYPYC_DECLARED_tuple_T2V88 +typedef struct tuple_T2V88 { + VecI64 f0; + int64_t f1; +} tuple_T2V88; +static tuple_T2V88 tuple_undefined_T2V88 = { { -1, NULL } , 0 }; +#endif + +typedef tuple_T2V88 VecI64PopResult; // vec[i64] operations + type objects (stored in a capsule) typedef struct _VecI64Features { @@ -112,10 +118,16 @@ typedef struct _VecI64Features { // iter? } VecI64Features; -typedef struct { - VecT vec; - PyObject *item; -} VecTPopResult; +#ifndef MYPYC_DECLARED_tuple_T2VOO +#define MYPYC_DECLARED_tuple_T2VOO +typedef struct tuple_T2VOO { + VecT f0; + PyObject *f1; +} tuple_T2VOO; +static tuple_T2VOO tuple_undefined_T2VOO = { { -1, NULL } , NULL }; +#endif + +typedef tuple_T2VOO VecTPopResult; // vec[T] operations + type object (stored in a capsule) // From 6f61090dd584aded27d8f5896d52ad11f9c7819e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jun 2023 16:09:25 +0100 Subject: [PATCH 091/208] Adjust borrowing/stealing a bit --- mypyc/lib-rt/vecs/vec_t_ext.c | 9 +++++++-- mypyc/lib-rt/vecs/vecs.h | 5 +---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index bd60b67c1cd4d..e6b70561352ce 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -133,6 +133,8 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecbufTExtItem item; if (Vec_T_Ext_UnboxItem(v, o, &item) < 0) return -1; + VEC_INCREF(item); + VEC_DECREF(v.buf->items[i]); v.buf->items[i] = item; return 0; } else { @@ -186,9 +188,10 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { return res; } -// Steals references to vec and x +// Steals reference to vec (but not x) VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { Py_ssize_t cap = VEC_CAP(vec); + VEC_INCREF(x); if (vec.len < cap) { vec.buf->items[vec.len] = x; vec.len++; @@ -198,8 +201,10 @@ VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { // TODO: Avoid initializing to zero here VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->optionals, vec.buf->depth); - if (VEC_IS_ERROR(new)) + if (VEC_IS_ERROR(new)) { + VEC_DECREF(x); return new; + } // Copy items to new vec. memcpy(new.buf->items, vec.buf->items, sizeof(VecbufTExtItem) * vec.len); // TODO: How to safely represent deleted items? diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 546a2be031070..78883df5b0daa 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -287,7 +287,7 @@ VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. -// Return a new reference. +// Return a *borrowed* reference. static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { int optionals = v.buf->optionals; if (item == Py_None && (optionals & 1)) { @@ -303,14 +303,12 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem if (o->vec.buf->item_type == v.buf->item_type) { unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); return 0; } } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { VecI64Object *o = (VecI64Object *)item; unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); return 0; } } else if (item->ob_type == &VecTExtType) { @@ -320,7 +318,6 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem && o->vec.buf->optionals == (optionals >> 1)) { unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; - Py_INCREF(unboxed->buf); return 0; } } From 2b23f1bbd4d9da27540ec0a57e77b57efb01f15d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 10 Jun 2023 16:47:13 +0100 Subject: [PATCH 092/208] Use magic constant for i64 etc. special item types in nested vecs This makes it a bit easier to work with them from mypyc-generated code. --- mypyc/lib-rt/vecs/vec_t_ext.c | 10 ++++++---- mypyc/lib-rt/vecs/vecs.c | 26 +++++++++++++++++++++----- mypyc/lib-rt/vecs/vecs.h | 13 ++++++++++--- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index e6b70561352ce..92cc69ebec74c 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -23,7 +23,8 @@ static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t option buf->item_type = item_type; buf->optionals = optionals; buf->depth = depth; - Py_INCREF(BUF_ITEM_TYPE(buf)); + if (!vec_is_magic_item_type(item_type)) + Py_INCREF(BUF_ITEM_TYPE(buf)); VecTExt res = { .buf = buf }; PyObject_GC_Track(buf); return res; @@ -314,7 +315,8 @@ VecTExt_dealloc(VecTObject *self) static int VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) { - Py_VISIT(BUF_ITEM_TYPE(self)); + if (!vec_is_magic_item_type(self->item_type)) + Py_VISIT(BUF_ITEM_TYPE(self)); for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_VISIT(self->items[i].buf); } @@ -324,7 +326,7 @@ VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) static int VecbufTExt_clear(VecbufTExtObject *self) { - if (self->item_type) { + if (self->item_type && !vec_is_magic_item_type(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; } @@ -339,7 +341,7 @@ VecbufTExt_dealloc(VecbufTExtObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecbufTExt_dealloc) - if (self->item_type) { + if (self->item_type && !vec_is_magic_item_type(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index dc7df8bfa8428..f4da661cb0718 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -50,14 +50,15 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) static int VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) { - Py_VISIT((PyObject *)(self->item_type & ~1)); + if (!vec_is_magic_item_type(self->item_type)) + Py_VISIT((PyObject *)(self->item_type & ~1)); return 0; } static void VecProxy_dealloc(VecProxy *self) { - if (self->item_type) { + if (self->item_type && !vec_is_magic_item_type(self->item_type)) { Py_DECREF((PyObject *)(self->item_type & ~1)); self->item_type = 0; } @@ -67,7 +68,11 @@ VecProxy_dealloc(VecProxy *self) PyObject *vec_type_to_str(size_t item_type, int32_t depth, int32_t optionals) { PyObject *item; if (depth == 0) - item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); + if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { + item = PyUnicode_FromFormat("i64"); + } else { + item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); + } else item = vec_type_to_str(item_type, depth - 1, optionals >> 1); @@ -144,6 +149,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } else { int32_t optionals = 0; int32_t depth = 0; + size_t item_type; if (!PyObject_TypeCheck(item, &PyType_Type)) { PyObject *it = extract_optional_item(item); if (it != NULL) { @@ -152,12 +158,22 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } if (item->ob_type == &VecProxyType) { VecProxy *p = (VecProxy *)item; - item = (PyObject *)p->item_type; + item_type = p->item_type & ~1; depth = p->depth + 1; optionals |= p->optionals << 1; } else if (!PyObject_TypeCheck(item, &PyType_Type)) { PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); return NULL; + } else { + item_type = (size_t)item; + } + } else { + if (item == (PyObject *)&VecI64Type) { + item_type = VEC_ITEM_TYPE_I64; + depth = 1; + // TODO: Check optionals? + } else { + item_type = (size_t)item; } } if (item == (PyObject *)&PyLong_Type @@ -171,7 +187,7 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (p == NULL) return NULL; Py_INCREF(item); - p->item_type = (size_t)item | ((optionals >> depth) & 1); + p->item_type = (size_t)item_type | ((optionals >> depth) & 1); p->depth = depth; p->optionals = optionals; PyObject_GC_Track(p); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 78883df5b0daa..32d5162c47f69 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -8,6 +8,13 @@ // use PyErr_Occurred() since this overlaps with valid integer values. #define MYPYC_INT_ERROR -113 +// Item type constants; must be even but not multiples of 4 +#define VEC_ITEM_TYPE_I64 2 + +inline size_t vec_is_magic_item_type(size_t item_type) { + return item_type & 2; +} + // Buffer objects @@ -305,7 +312,7 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem unboxed->buf = (PyObject *)o->vec.buf; return 0; } - } else if (item->ob_type == &VecI64Type && v.buf->item_type == (size_t)I64TypeObj) { + } else if (item->ob_type == &VecI64Type && v.buf->item_type == VEC_ITEM_TYPE_I64) { VecI64Object *o = (VecI64Object *)item; unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; @@ -336,8 +343,8 @@ static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { return Vec_T_Ext_Box(v); } else { // Item is a non-nested vec - void *item_type = (void *)(v.buf->item_type & ~1); - if (item_type == I64TypeObj) { + size_t item_type = v.buf->item_type & ~1; + if (item_type == VEC_ITEM_TYPE_I64) { // vec[i64] VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; return Vec_I64_Box(v); From 6495613edde366f9c686de50a70c977c51ebab13 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 11 Jun 2023 17:45:26 +0100 Subject: [PATCH 093/208] Add missing unboxing C API functions (untested) --- mypyc/lib-rt/vecs/vec_t.c | 15 ++++++++++++++- mypyc/lib-rt/vecs/vec_t_ext.c | 17 ++++++++++++++++- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 9ad6d47f53704..b455e6020697e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -31,6 +31,19 @@ PyObject *Vec_T_Box(VecT vec) { return (PyObject *)obj; } +VecT Vec_T_Unbox(PyObject *obj, size_t item_type) { + if (obj->ob_type == &VecTType) { + VecT result = ((VecTObject *)obj)->vec; + if (result.buf->item_type == item_type) { + VEC_INCREF(result); // TODO: Should we borrow instead? + return result; + } + } + // TODO: Better error message, with name of type + PyErr_SetString(PyExc_TypeError, "vec[t] expected"); + return Vec_T_Error(); +} + VecT Vec_T_New(Py_ssize_t size, size_t item_type) { VecT vec = vec_t_alloc(size, item_type); if (VEC_IS_ERROR(vec)) @@ -389,7 +402,7 @@ VecTFeatures TFeatures = { &VecbufTType, Vec_T_New, Vec_T_Box, - NULL, // TODO: unbox + Vec_T_Unbox, Vec_T_Append, Vec_T_Pop, Vec_T_Remove, diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 92cc69ebec74c..19643b526eb55 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -39,6 +39,21 @@ PyObject *Vec_T_Ext_Box(VecTExt vec) { return (PyObject *)obj; } +VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, int optionals, int depth) { + if (obj->ob_type == &VecTExtType) { + VecTExt result = ((VecTExtObject *)obj)->vec; + if (result.buf->item_type == item_type + && result.buf->optionals == optionals + && result.buf->depth == depth) { + VEC_INCREF(result); // TODO: Should we borrow instead? + return result; + } + } + // TODO: Better error message, with name of type + PyErr_SetString(PyExc_TypeError, "vec[t] expected"); + return Vec_T_Ext_Error(); +} + VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { VecTExt vec = vec_t_ext_alloc(size, item_type, optionals, depth); if (VEC_IS_ERROR(vec)) @@ -446,7 +461,7 @@ VecTExtFeatures TExtFeatures = { &VecbufTExtType, Vec_T_Ext_New, Vec_T_Ext_Box, - NULL, // TODO: unbox + Vec_T_Ext_Unbox, Vec_T_Ext_Append, Vec_T_Ext_Pop, Vec_T_Ext_Remove, diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 32d5162c47f69..2179fee6414e8 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -144,7 +144,7 @@ typedef struct _VecTFeatures { PyTypeObject *buf_type; VecT (*alloc)(Py_ssize_t, size_t); PyObject *(*box)(VecT); - VecT (*unbox)(size_t); + VecT (*unbox)(PyObject *, size_t); VecT (*append)(VecT, PyObject *); VecTPopResult (*pop)(VecT, Py_ssize_t); VecT (*remove)(VecT, PyObject *); @@ -169,7 +169,7 @@ typedef struct _VecTExtFeatures { PyTypeObject *buf_type; VecTExt (*alloc)(Py_ssize_t, size_t, int optionals, int depth); PyObject *(*box)(VecTExt); - VecTExt (*unbox)(size_t, int optionals, int depth); + VecTExt (*unbox)(PyObject *, size_t, int optionals, int depth); VecTExt (*append)(VecTExt, VecbufTExtItem); VecTExtPopResult (*pop)(VecTExt, Py_ssize_t); VecTExt (*remove)(VecTExt, VecbufTExtItem); From 14917299cfff1368561b319631d413c267a96762 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jun 2023 16:24:21 +0100 Subject: [PATCH 094/208] Fix remove --- mypyc/lib-rt/vecs/vec_t.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index b455e6020697e..a96d54b019cdc 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -207,12 +207,13 @@ VecT Vec_T_Remove(VecT v, PyObject *arg) { match = itemcmp; } if (match) { - Py_CLEAR(items[i]); - for (; i < v.len - 1; i++) { - items[i] = items[i + 1]; - } - if (v.len > 0) + if (i < v.len - 1) { + Py_CLEAR(items[i]); + for (; i < v.len - 1; i++) { + items[i] = items[i + 1]; + } Py_XINCREF(items[v.len - 1]); + } v.len--; VEC_INCREF(v); return v; From 355326fd93e4487b1db7237776b99013b5c45a5c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 17 Jun 2023 16:25:14 +0100 Subject: [PATCH 095/208] Fix remove --- mypyc/lib-rt/vecs/vec_t_ext.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 19643b526eb55..a5a2a331e98fc 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -260,12 +260,13 @@ VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { match = itemcmp; } if (match) { - Py_CLEAR(items[i].buf); - for (; i < self.len - 1; i++) { - items[i] = items[i + 1]; - } - if (self.len > 0) + if (i < self.len - 1) { + Py_CLEAR(items[i].buf); + for (; i < self.len - 1; i++) { + items[i] = items[i + 1]; + } Py_XINCREF(items[self.len - 1].buf); + } self.len--; Py_INCREF(self.buf); Py_DECREF(boxed_arg); From a3b9da490119a8e68be342adb0b806f25a2e264b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 13:40:43 +0100 Subject: [PATCH 096/208] Fix slicing overflow checks --- mypyc/lib-rt/vecs/vec_i64.c | 2 ++ mypyc/lib-rt/vecs/vec_t.c | 2 ++ mypyc/lib-rt/vecs/vec_t_ext.c | 2 ++ 3 files changed, 6 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index c9855bee27aa5..27e48b1f5b64c 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -125,6 +125,8 @@ VecI64 Vec_I64_Slice(VecI64 vec, int64_t start, int64_t end) { end = start; if (start < 0) start = 0; + if (start >= vec.len) + start = vec.len; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index a96d54b019cdc..c541b6ac7487d 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -81,6 +81,8 @@ VecT Vec_T_Slice(VecT vec, int64_t start, int64_t end) { end = start; if (start < 0) start = 0; + if (start >= vec.len) + start = vec.len; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index a5a2a331e98fc..0fdba525da462 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -90,6 +90,8 @@ VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { end = start; if (start < 0) start = 0; + if (start >= vec.len) + start = vec.len; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; From 1fae06d7a5b84260da2140ed0ec4ab019e3745d2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 8 Jul 2023 14:10:20 +0100 Subject: [PATCH 097/208] Fix slicing --- mypyc/lib-rt/vecs/vec_i64.c | 4 ++-- mypyc/lib-rt/vecs/vec_t.c | 4 ++-- mypyc/lib-rt/vecs/vec_t_ext.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 27e48b1f5b64c..0f536044fc3dc 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -121,12 +121,12 @@ VecI64 Vec_I64_Slice(VecI64 vec, int64_t start, int64_t end) { start += vec.len; if (end < 0) end += vec.len; - if (end < start) - end = start; if (start < 0) start = 0; if (start >= vec.len) start = vec.len; + if (end < start) + end = start; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index c541b6ac7487d..2e643174067d5 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -77,12 +77,12 @@ VecT Vec_T_Slice(VecT vec, int64_t start, int64_t end) { start += vec.len; if (end < 0) end += vec.len; - if (end < start) - end = start; if (start < 0) start = 0; if (start >= vec.len) start = vec.len; + if (end < start) + end = start; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 0fdba525da462..d0934bb641001 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -86,12 +86,12 @@ VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { start += vec.len; if (end < 0) end += vec.len; - if (end < start) - end = start; if (start < 0) start = 0; if (start >= vec.len) start = vec.len; + if (end < start) + end = start; if (end > vec.len) end = vec.len; int64_t slicelength = end - start; From c1ebbe98080c5e21e95a1436af81fc18dea07e93 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 12:24:14 +0100 Subject: [PATCH 098/208] Only the non-vec item type in a nested vec can be optional This simplified the implementation significantly, and use cases like `vec[vec[C] | None]` seem pretty marginal. --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_t_ext.c | 36 +++++++++++---------------- mypyc/lib-rt/vecs/vecs.c | 46 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vecs.h | 39 ++++++++++++----------------- 5 files changed, 54 insertions(+), 71 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 0f536044fc3dc..9f54cd7068da5 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -103,7 +103,7 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } PyObject *vec_i64_repr(PyObject *self) { - return vec_repr(self, (size_t)I64TypeObj, 0, 0, 1); + return vec_repr(self, (size_t)I64TypeObj, 0, 1); } PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 2e643174067d5..711fdd51b7467 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -57,7 +57,7 @@ VecT Vec_T_New(Py_ssize_t size, size_t item_type) { PyObject *vec_t_repr(PyObject *self) { VecTObject *v = (VecTObject *)self; - return vec_repr(self, v->vec.buf->item_type, 0, 0, 1); + return vec_repr(self, v->vec.buf->item_type, 0, 1); } PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index d0934bb641001..df1d979e1a2e2 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -14,14 +14,13 @@ static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { } -// Alloc a partially initialized vec. Caller *must* initialize len and buf->items. -static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, int32_t optionals, - int32_t depth) { +// Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the +// return value. +static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); if (buf == NULL) return Vec_T_Ext_Error(); buf->item_type = item_type; - buf->optionals = optionals; buf->depth = depth; if (!vec_is_magic_item_type(item_type)) Py_INCREF(BUF_ITEM_TYPE(buf)); @@ -39,12 +38,10 @@ PyObject *Vec_T_Ext_Box(VecTExt vec) { return (PyObject *)obj; } -VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, int optionals, int depth) { +VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { if (obj->ob_type == &VecTExtType) { VecTExt result = ((VecTExtObject *)obj)->vec; - if (result.buf->item_type == item_type - && result.buf->optionals == optionals - && result.buf->depth == depth) { + if (result.buf->item_type == item_type && result.buf->depth == depth) { VEC_INCREF(result); // TODO: Should we borrow instead? return result; } @@ -54,8 +51,8 @@ VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, int optionals, int dept return Vec_T_Ext_Error(); } -VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth) { - VecTExt vec = vec_t_ext_alloc(size, item_type, optionals, depth); +VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, size_t depth) { + VecTExt vec = vec_t_ext_alloc(size, item_type, depth); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < size; i++) { @@ -68,7 +65,7 @@ VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int3 PyObject *vec_t_ext_repr(PyObject *self) { VecTExt v = ((VecTExtObject *)self)->vec; - return vec_repr(self, v.buf->item_type, v.buf->depth, v.buf->optionals, 1); + return vec_repr(self, v.buf->item_type, v.buf->depth, 1); } PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { @@ -95,8 +92,7 @@ VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->optionals, - vec.buf->depth); + VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -125,8 +121,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->optionals, - vec.buf->depth); + VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -166,8 +161,7 @@ static PyObject *compare_vec_t_ext_eq(VecTExt x, VecTExt y, int op) { PyObject *res; if (x.len != y.len || x.buf->item_type != y.buf->item_type - || x.buf->depth != y.buf->depth - || x.buf->optionals != y.buf->optionals) { + || x.buf->depth != y.buf->depth) { cmp = 0; } else { for (Py_ssize_t i = 0; i < x.len; i++) { @@ -217,8 +211,7 @@ VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->optionals, - vec.buf->depth); + VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(new)) { VEC_DECREF(x); return new; @@ -422,9 +415,8 @@ PyTypeObject VecTExtType = { // TODO: free }; -PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t depth, - PyObject *iterable) { - VecTExt v = vec_t_ext_alloc(0, item_type, optionals, depth); +PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { + VecTExt v = vec_t_ext_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index f4da661cb0718..0365235709aa8 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -13,9 +13,9 @@ PyTypeObject *I64TypeObj; typedef struct { PyObject_HEAD // Tagged pointer to PyTypeObject *, lowest bit set for optional item type + // Can also be one of magic VEC_ITEM_TYPE_* constants size_t item_type; - int32_t depth; // Number of nested VecTExt or VecT types - int32_t optionals; // Flags for optional types on each nesting level + size_t depth; // Number of nested VecTExt or VecT types } VecProxy; static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) @@ -37,12 +37,12 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } } else { if (init == NULL) { - VecTExt vec = Vec_T_Ext_New(0, p->item_type, p->optionals, p->depth); + VecTExt vec = Vec_T_Ext_New(0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; return Vec_T_Ext_Box(vec); } else { - return Vec_T_Ext_FromIterable(p->item_type, p->optionals, p->depth, init); + return Vec_T_Ext_FromIterable(p->item_type, p->depth, init); } } } @@ -65,21 +65,20 @@ VecProxy_dealloc(VecProxy *self) PyObject_GC_Del(self); } -PyObject *vec_type_to_str(size_t item_type, int32_t depth, int32_t optionals) { +PyObject *vec_type_to_str(size_t item_type, size_t depth) { PyObject *item; if (depth == 0) if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { item = PyUnicode_FromFormat("i64"); } else { item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); + if (item_type & 1) { + item = PyUnicode_FromFormat("%U | None", item); + } } else - item = vec_type_to_str(item_type, depth - 1, optionals >> 1); + item = vec_type_to_str(item_type, depth - 1); - // TODO: clean up - if ((optionals & 1) || (item_type & 1)) { - item = PyUnicode_FromFormat("%U | None", item); - } return PyUnicode_FromFormat("vec[%U]", item); } @@ -91,7 +90,7 @@ PyObject *VecProxy_repr(PyObject *self) { PyObject *sep = Py_BuildValue("s", ""); PyList_Append(l, prefix); VecProxy *v = (VecProxy *)self; - PyList_Append(l, vec_type_to_str(v->item_type, v->depth, v->optionals)); + PyList_Append(l, vec_type_to_str(v->item_type, v->depth)); PyList_Append(l, suffix); return PyUnicode_Join(sep, l); } @@ -147,25 +146,28 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; } else { - int32_t optionals = 0; - int32_t depth = 0; + size_t depth = 0; size_t item_type; + int optional = 0; if (!PyObject_TypeCheck(item, &PyType_Type)) { PyObject *it = extract_optional_item(item); if (it != NULL) { - optionals = 1; + optional = 1; item = it; } if (item->ob_type == &VecProxyType) { + if (optional) { + PyErr_SetString(PyExc_TypeError, "optional type not expected in vec[...]"); + return NULL; + } VecProxy *p = (VecProxy *)item; - item_type = p->item_type & ~1; + item_type = p->item_type; depth = p->depth + 1; - optionals |= p->optionals << 1; } else if (!PyObject_TypeCheck(item, &PyType_Type)) { PyErr_SetString(PyExc_TypeError, "type object expected in vec[...]"); return NULL; } else { - item_type = (size_t)item; + item_type = (size_t)item | optional; } } else { if (item == (PyObject *)&VecI64Type) { @@ -187,9 +189,8 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (p == NULL) return NULL; Py_INCREF(item); - p->item_type = (size_t)item_type | ((optionals >> depth) & 1); + p->item_type = item_type; p->depth = depth; - p->optionals = optionals; PyObject_GC_Track(p); return (PyObject *)p; } @@ -209,8 +210,7 @@ PyTypeObject VecGenericType = { .tp_methods = vec_methods, }; -PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optionals, - int verbose) { +PyObject *vec_repr(PyObject *vec, size_t item_type, size_t depth, int verbose) { // TODO: Check for errors // TODO: Refcount handling PyObject *l = Py_BuildValue("[]"); @@ -220,7 +220,7 @@ PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optio PyObject *sep = Py_BuildValue("s", ""); PyObject *comma = Py_BuildValue("s", ", "); if (verbose) { - prefix = vec_type_to_str(item_type, depth, optionals); + prefix = vec_type_to_str(item_type, depth); mid = Py_BuildValue("s", "(["); suffix = Py_BuildValue("s", "])"); } else { @@ -239,7 +239,7 @@ PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optio if (depth == 0 || it == Py_None) { r = PyObject_Repr(it); } else { - r = vec_repr(it, item_type, depth - 1, optionals >> 1, 0); + r = vec_repr(it, item_type, depth - 1, 0); } if (r == NULL) return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 2179fee6414e8..fc2fd03e4ac78 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -8,7 +8,7 @@ // use PyErr_Occurred() since this overlaps with valid integer values. #define MYPYC_INT_ERROR -113 -// Item type constants; must be even but not multiples of 4 +// Item type constants; must be even but not multiples of 4 (2 + 4 * n) #define VEC_ITEM_TYPE_I64 2 inline size_t vec_is_magic_item_type(size_t item_type) { @@ -41,10 +41,11 @@ typedef struct _VecbufTExtItem { // Nested vec type: vec[vec[...]], vec[vec[...] | None], etc. typedef struct _VecbufTExtObject { PyObject_VAR_HEAD - // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. + // Tagged pointer to PyTypeObject *. Lowest bit is set for optional item type. + // The second lowest bit is set for a packed item type (VEC_ITEM_TYPE_*). size_t item_type; - int32_t depth; // Number of nested VecTExt or VecT types - int32_t optionals; // Flags for optional types on each nesting level + // Number of nested vec types (of any kind, at least 1) + size_t depth; VecbufTExtItem items[1]; } VecbufTExtObject; @@ -167,9 +168,9 @@ typedef struct { typedef struct _VecTExtFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecTExt (*alloc)(Py_ssize_t, size_t, int optionals, int depth); + VecTExt (*alloc)(Py_ssize_t, size_t, size_t depth); PyObject *(*box)(VecTExt); - VecTExt (*unbox)(PyObject *, size_t, int optionals, int depth); + VecTExt (*unbox)(PyObject *, size_t, size_t depth); VecTExt (*append)(VecTExt, VecbufTExtItem); VecTExtPopResult (*pop)(VecTExt, Py_ssize_t); VecTExt (*remove)(VecTExt, VecbufTExtItem); @@ -268,15 +269,14 @@ static inline int VecTExt_Check(PyObject *o) { static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { // TODO: vec[i64] item type - if (it == Py_None && (v.buf->optionals & 1)) { + if (it == Py_None && (v.buf->item_type & 1)) { return 1; } else if (v.buf->depth == 1 && it->ob_type == &VecTType && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type) { return 1; } else if (it->ob_type == &VecTExtType && ((VecTExtObject *)it)->vec.buf->depth == v.buf->depth - 1 - && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type - && ((VecTExtObject *)it)->vec.buf->optionals == (v.buf->optionals >> 1)) { + && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type) { return 1; } else { // TODO: better error message @@ -285,9 +285,8 @@ static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { } } -VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, int32_t optionals, int32_t depth); -PyObject *Vec_T_Ext_FromIterable(size_t item_type, int32_t optionals, int32_t depth, - PyObject *iterable); +VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, size_t depth); +PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable); PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); @@ -296,13 +295,7 @@ VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { - int optionals = v.buf->optionals; - if (item == Py_None && (optionals & 1)) { - unboxed->len = -1; - unboxed->buf = NULL; - return 0; - } - int depth = v.buf->depth; + size_t depth = v.buf->depth; if (depth == 1) { // TODO: vec[i64] if (item->ob_type == &VecTType) { @@ -321,8 +314,7 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem } else if (item->ob_type == &VecTExtType) { VecTExtObject *o = (VecTExtObject *)item; if (o->vec.buf->depth == v.buf->depth - 1 - && o->vec.buf->item_type == v.buf->item_type - && o->vec.buf->optionals == (optionals >> 1)) { + && o->vec.buf->item_type == v.buf->item_type) { unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; return 0; @@ -366,9 +358,8 @@ static inline int check_float_error(PyObject *o) { return 0; } -PyObject *vec_type_to_str(size_t item_type, int32_t depth, int32_t optionals); -PyObject *vec_repr(PyObject *vec, size_t item_type, int32_t depth, int32_t optionals, - int verbose); +PyObject *vec_type_to_str(size_t item_type, size_t depth); +PyObject *vec_repr(PyObject *vec, size_t item_type, size_t depth, int verbose); PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); From bdcd956c4e25a4b96f2d9847d3c01161bdd346e9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 13:53:58 +0100 Subject: [PATCH 099/208] Allow empty vec[t] to have NULL buf This avoids an allocation when constructing an empty vec[t]. --- mypyc/lib-rt/vecs/vec_t.c | 53 ++++++++++++++++++++++++++++++--------- mypyc/lib-rt/vecs/vecs.c | 14 +++++------ mypyc/lib-rt/vecs/vecs.h | 21 ++++++++-------- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 711fdd51b7467..4aa5354cede3b 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -10,19 +10,39 @@ #include #include "vecs.h" -// Alloc a partially initialized vec. Caller *must* initialize len and buf->items. -static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { +static inline VecbufTObject *alloc_buf(Py_ssize_t size, size_t item_type) { VecbufTObject *buf = PyObject_GC_NewVar(VecbufTObject, &VecbufTType, size); if (buf == NULL) - return Vec_T_Error(); + return NULL; buf->item_type = item_type; Py_INCREF(BUF_ITEM_TYPE(buf)); - VecT res = { .buf = buf }; PyObject_GC_Track(buf); - return res; + return buf; } -PyObject *Vec_T_Box(VecT vec) { +// Alloc a partially initialized vec. Caller *must* immediately initialize len (and buf->items +// if size > 0). +static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { + VecbufTObject *buf; + + if (size == 0) { + buf = NULL; + } else { + buf = alloc_buf(size, item_type); + if (buf == NULL) + return Vec_T_Error(); + } + return (VecT) { .buf = buf }; +} + +PyObject *Vec_T_Box(VecT vec, size_t item_type) { + // An unboxed empty vec may have a NULL buf, but a boxed vec must have it + // allocated, since it contains the item type + if (vec.buf == NULL) { + vec.buf = alloc_buf(0, item_type); + if (vec.buf == NULL) + return NULL; + } VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); if (obj == NULL) return NULL; @@ -128,7 +148,7 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return Vec_T_Box(res); + return Vec_T_Box(res, vec.buf->item_type); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -138,7 +158,7 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecT v = ((VecTObject *)self)->vec; - if (!VecT_ItemCheck(v, o)) + if (!VecT_ItemCheck(v, o, v.buf->item_type)) return -1; if ((size_t)i < (size_t)v.len) { Py_INCREF(o); @@ -171,7 +191,16 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecT Vec_T_Append(VecT vec, PyObject *x) { +VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type) { + if (vec.buf == NULL) { + VecT new = vec_t_alloc(1, item_type); + if (VEC_IS_ERROR(new)) + return new; + Py_INCREF(x); + new.len = 1; + new.buf->items[0] = x; + return new; + } Py_ssize_t cap = VEC_CAP(vec); Py_INCREF(x); if (vec.len < cap) { @@ -378,13 +407,13 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { - if (!VecT_ItemCheck(v, item)) { + if (!VecT_ItemCheck(v, item, item_type)) { Py_DECREF(iter); VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = Vec_T_Append(v, item); + v = Vec_T_Append(v, item, item_type); Py_DECREF(item); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); @@ -397,7 +426,7 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { VEC_DECREF(v); return NULL; } - return Vec_T_Box(v); + return Vec_T_Box(v, item_type); } VecTFeatures TFeatures = { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 0365235709aa8..5350e345fb256 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -31,7 +31,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) VecT vec = Vec_T_New(0, p->item_type); if (VEC_IS_ERROR(vec)) return NULL; - return Vec_T_Box(vec); + return Vec_T_Box(vec, p->item_type); } else { return Vec_T_FromIterable(p->item_type, init); } @@ -356,14 +356,14 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return Vec_I64_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; - if (!VecT_ItemCheck(v, item)) { + if (!VecT_ItemCheck(v, item, v.buf->item_type)) { return NULL; } VEC_INCREF(v); - v = Vec_T_Append(v, item); + v = Vec_T_Append(v, item, v.buf->item_type); if (VEC_IS_ERROR(v)) return NULL; - return Vec_T_Box(v); + return Vec_T_Box(v, v.buf->item_type); } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; @@ -403,14 +403,14 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) return Vec_I64_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; - if (!VecT_ItemCheck(v, item)) { + if (!VecT_ItemCheck(v, item, v.buf->item_type)) { return NULL; } VEC_INCREF(v); v = Vec_T_Remove(v, item); if (VEC_IS_ERROR(v)) return NULL; - return Vec_T_Box(v); + return Vec_T_Box(v, v.buf->item_type); } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; @@ -456,7 +456,7 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_T_Box(r.f0); + result_item0 = Vec_T_Box(r.f0, v.buf->item_type); if (result_item0 == NULL) { Py_DECREF(r.f1); return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index fc2fd03e4ac78..73fb39b051e7b 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -144,9 +144,9 @@ typedef struct _VecTFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecT (*alloc)(Py_ssize_t, size_t); - PyObject *(*box)(VecT); + PyObject *(*box)(VecT, size_t); VecT (*unbox)(PyObject *, size_t); - VecT (*append)(VecT, PyObject *); + VecT (*append)(VecT, PyObject *, size_t); VecTPopResult (*pop)(VecT, Py_ssize_t); VecT (*remove)(VecT, PyObject *); // TODO: Py_ssize_t @@ -189,7 +189,8 @@ typedef struct { } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) -#define BUF_ITEM_TYPE(b) ((PyTypeObject *)((b)->item_type & ~1)) +#define ITEM_TYPE(t) ((PyTypeObject *)((t) & ~1)) +#define BUF_ITEM_TYPE(b) ITEM_TYPE((b)->item_type) #define VEC_CAP(v) ((v).buf->ob_base.ob_size) #define VEC_IS_ERROR(v) ((v).len < 0) #define VEC_DECREF(v) Py_XDECREF((v).buf) @@ -242,10 +243,10 @@ static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; } -static inline int VecT_ItemCheck(VecT v, PyObject *item) { - if (PyObject_TypeCheck(item, BUF_ITEM_TYPE(v.buf))) { +static inline int VecT_ItemCheck(VecT v, PyObject *item, size_t item_type) { + if (PyObject_TypeCheck(item, ITEM_TYPE(item_type))) { return 1; - } else if ((v.buf->item_type & 1) && item == Py_None) { + } else if ((item_type & 1) && item == Py_None) { return 1; } else { // TODO: better error message @@ -256,8 +257,8 @@ static inline int VecT_ItemCheck(VecT v, PyObject *item) { VecT Vec_T_New(Py_ssize_t size, size_t item_type); PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); -PyObject *Vec_T_Box(VecT); -VecT Vec_T_Append(VecT vec, PyObject *x); +PyObject *Vec_T_Box(VecT vec, size_t item_type); +VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type); VecT Vec_T_Remove(VecT vec, PyObject *x); VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index); @@ -335,7 +336,7 @@ static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { return Vec_T_Ext_Box(v); } else { // Item is a non-nested vec - size_t item_type = v.buf->item_type & ~1; + size_t item_type = v.buf->item_type; if (item_type == VEC_ITEM_TYPE_I64) { // vec[i64] VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; @@ -343,7 +344,7 @@ static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; - return Vec_T_Box(v); + return Vec_T_Box(v, item_type); } } } From e310bf8eed8754fc5e41536ea39900fb8fa0d89d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 19 Aug 2023 15:24:01 +0100 Subject: [PATCH 100/208] Allow specifying capacity when allocating a vec --- mypyc/lib-rt/vecs/vec_i64.c | 10 ++++++---- mypyc/lib-rt/vecs/vec_t.c | 8 +++++--- mypyc/lib-rt/vecs/vec_t_ext.c | 8 +++++--- mypyc/lib-rt/vecs/vecs.c | 4 ++-- mypyc/lib-rt/vecs/vecs.h | 10 +++++----- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 9f54cd7068da5..ade0f61f11433 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -43,11 +43,13 @@ VecI64 Vec_I64_Unbox(PyObject *obj) { } } -VecI64 Vec_I64_New(Py_ssize_t size) { - VecI64 vec = vec_i64_alloc(size); +VecI64 Vec_I64_New(Py_ssize_t size, Py_ssize_t cap) { + if (cap < size) + size = cap; + VecI64 vec = vec_i64_alloc(cap); if (VEC_IS_ERROR(vec)) return vec; - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < cap; i++) { vec.buf->items[i] = 0; } vec.len = size; @@ -96,7 +98,7 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return NULL; } if (init == NULL) { - return Vec_I64_Box(Vec_I64_New(0)); + return Vec_I64_Box(Vec_I64_New(0, 0)); } else { return (PyObject *)Vec_I64_FromIterable(init); } diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 4aa5354cede3b..3b5a77df2028d 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -64,11 +64,13 @@ VecT Vec_T_Unbox(PyObject *obj, size_t item_type) { return Vec_T_Error(); } -VecT Vec_T_New(Py_ssize_t size, size_t item_type) { - VecT vec = vec_t_alloc(size, item_type); +VecT Vec_T_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { + if (cap < size) + cap = size; + VecT vec = vec_t_alloc(cap, item_type); if (VEC_IS_ERROR(vec)) return vec; - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < cap; i++) { vec.buf->items[i] = NULL; } vec.len = size; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index df1d979e1a2e2..d69a92832ecfe 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -51,11 +51,13 @@ VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { return Vec_T_Ext_Error(); } -VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, size_t depth) { - VecTExt vec = vec_t_ext_alloc(size, item_type, depth); +VecTExt Vec_T_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { + if (cap < size) + cap = size; + VecTExt vec = vec_t_ext_alloc(cap, item_type, depth); if (VEC_IS_ERROR(vec)) return vec; - for (Py_ssize_t i = 0; i < size; i++) { + for (Py_ssize_t i = 0; i < cap; i++) { vec.buf->items[i].len = -1; vec.buf->items[i].buf = NULL; } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 5350e345fb256..67b908d61da54 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -28,7 +28,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) VecProxy *p = (VecProxy *)self; if (p->depth == 0) { if (init == NULL) { - VecT vec = Vec_T_New(0, p->item_type); + VecT vec = Vec_T_New(0, 0, p->item_type); if (VEC_IS_ERROR(vec)) return NULL; return Vec_T_Box(vec, p->item_type); @@ -37,7 +37,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } } else { if (init == NULL) { - VecTExt vec = Vec_T_Ext_New(0, p->item_type, p->depth); + VecTExt vec = Vec_T_Ext_New(0, 0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; return Vec_T_Ext_Box(vec); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 73fb39b051e7b..91493f3fcf5ef 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -112,7 +112,7 @@ typedef tuple_T2V88 VecI64PopResult; typedef struct _VecI64Features { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecI64 (*alloc)(Py_ssize_t); + VecI64 (*alloc)(Py_ssize_t, Py_ssize_t); PyObject *(*box)(VecI64); VecI64 (*unbox)(PyObject *); VecI64 (*append)(VecI64, int64_t); @@ -143,7 +143,7 @@ typedef tuple_T2VOO VecTPopResult; typedef struct _VecTFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecT (*alloc)(Py_ssize_t, size_t); + VecT (*alloc)(Py_ssize_t, Py_ssize_t, size_t); PyObject *(*box)(VecT, size_t); VecT (*unbox)(PyObject *, size_t); VecT (*append)(VecT, PyObject *, size_t); @@ -168,7 +168,7 @@ typedef struct { typedef struct _VecTExtFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecTExt (*alloc)(Py_ssize_t, size_t, size_t depth); + VecTExt (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); PyObject *(*box)(VecTExt); VecTExt (*unbox)(PyObject *, size_t, size_t depth); VecTExt (*append)(VecTExt, VecbufTExtItem); @@ -255,7 +255,7 @@ static inline int VecT_ItemCheck(VecT v, PyObject *item, size_t item_type) { } } -VecT Vec_T_New(Py_ssize_t size, size_t item_type); +VecT Vec_T_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type); PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); PyObject *Vec_T_Box(VecT vec, size_t item_type); VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type); @@ -286,7 +286,7 @@ static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { } } -VecTExt Vec_T_Ext_New(Py_ssize_t size, size_t item_type, size_t depth); +VecTExt Vec_T_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable); PyObject *Vec_T_Ext_Box(VecTExt); VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); From 49d06afe2817df3f32734fabcfdc6d7c27206168 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 27 Aug 2023 16:47:24 +0100 Subject: [PATCH 101/208] Fix segfault --- mypyc/lib-rt/vecs/vecs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 91493f3fcf5ef..4e4be445971b2 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -329,7 +329,7 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { if (item.len < 0) Py_RETURN_NONE; - Py_INCREF(item.buf); + Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; From 626aa6bd72539383b56342fdce4311f6faae0e42 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 28 Aug 2023 11:18:55 +0100 Subject: [PATCH 102/208] Change return type of nested vec pop --- mypyc/lib-rt/vecs/vec_t_ext.c | 10 +++++----- mypyc/lib-rt/vecs/vecs.c | 12 ++++++------ mypyc/lib-rt/vecs/vecs.h | 15 +++++++++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index d69a92832ecfe..72880bb42f02e 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -283,21 +283,21 @@ VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.vec = Vec_T_Ext_Error(); - result.item.len = 0; - result.item.buf = NULL; + result.f0 = Vec_T_Ext_Error(); + result.f1.len = 0; + result.f1.buf = NULL; return result; } VecbufTExtItem *items = v.buf->items; - result.item = items[index]; + result.f1 = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; if (v.len > 0) Py_XINCREF(items[v.len - 1].buf); v.len--; VEC_INCREF(v); - result.vec = v; + result.f0 = v; return result; } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 67b908d61da54..63bc67c98790d 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -466,20 +466,20 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) VecTExt v = ((VecTExtObject *)vec)->vec; VecTExtPopResult r; r = Vec_T_Ext_Pop(v, index); - if (VEC_IS_ERROR(r.vec)) + if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_T_Ext_Box(r.vec); + result_item0 = Vec_T_Ext_Box(r.f0); if (result_item0 == NULL) { - Py_DECREF(r.vec.buf); - Py_DECREF(r.item.buf); + Py_DECREF(r.f0.buf); + Py_DECREF(r.f1.buf); return NULL; } - result_item1 = Vec_T_Ext_BoxItem(r.vec, r.item); + result_item1 = Vec_T_Ext_BoxItem(r.f0, r.f1); if (result_item1 == NULL) { Py_DECREF(result_item0); - Py_DECREF(r.item.buf); + Py_DECREF(r.f1.buf); return NULL; } } else { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 4e4be445971b2..7397c13b4addf 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -157,10 +157,17 @@ typedef struct _VecTFeatures { // iter? } VecTFeatures; -typedef struct { - VecTExt vec; - VecbufTExtItem item; -} VecTExtPopResult; + +#ifndef MYPYC_DECLARED_tuple_T2VvVi +#define MYPYC_DECLARED_tuple_T2VvVi +typedef struct tuple_T2VvVi { + VecTExt f0; + VecbufTExtItem f1; +} tuple_T2VvVi; +static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; +#endif + +typedef tuple_T2VvVi VecTExtPopResult; // vec[T] operations for complex item types + type object (stored in a capsule) // From 727d6ce173c98017751209838df16df803838f0b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 28 Aug 2023 11:24:32 +0100 Subject: [PATCH 103/208] Add conversion functions from VecbufTExtItem --- mypyc/lib-rt/vecs/vec_i64.c | 5 +++++ mypyc/lib-rt/vecs/vec_t.c | 5 +++++ mypyc/lib-rt/vecs/vec_t_ext.c | 5 +++++ mypyc/lib-rt/vecs/vecs.h | 3 +++ 4 files changed, 18 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index ade0f61f11433..d145eef4f7ba0 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -43,6 +43,10 @@ VecI64 Vec_I64_Unbox(PyObject *obj) { } } +VecI64 Vec_I64_ConvertFromNested(VecbufTExtItem item) { + return (VecI64) { item.len, (VecbufI64Object *)item.buf }; +} + VecI64 Vec_I64_New(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; @@ -332,6 +336,7 @@ VecI64Features I64Features = { Vec_I64_New, Vec_I64_Box, Vec_I64_Unbox, + Vec_I64_ConvertFromNested, Vec_I64_Append, Vec_I64_Pop, Vec_I64_Remove, diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3b5a77df2028d..b20b2be7327c9 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -64,6 +64,10 @@ VecT Vec_T_Unbox(PyObject *obj, size_t item_type) { return Vec_T_Error(); } +VecT Vec_T_ConvertFromNested(VecbufTExtItem item) { + return (VecT) { item.len, (VecbufTObject *)item.buf }; +} + VecT Vec_T_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { if (cap < size) cap = size; @@ -437,6 +441,7 @@ VecTFeatures TFeatures = { Vec_T_New, Vec_T_Box, Vec_T_Unbox, + Vec_T_ConvertFromNested, Vec_T_Append, Vec_T_Pop, Vec_T_Remove, diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 72880bb42f02e..1d64f3be26753 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -51,6 +51,10 @@ VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { return Vec_T_Ext_Error(); } +VecTExt Vec_T_Ext_ConvertFromNested(VecbufTExtItem item) { + return (VecTExt) { item.len, (VecbufTExtObject *)item.buf }; +} + VecTExt Vec_T_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; @@ -459,6 +463,7 @@ VecTExtFeatures TExtFeatures = { Vec_T_Ext_New, Vec_T_Ext_Box, Vec_T_Ext_Unbox, + Vec_T_Ext_ConvertFromNested, Vec_T_Ext_Append, Vec_T_Ext_Pop, Vec_T_Ext_Remove, diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 7397c13b4addf..ee0f278292057 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -115,6 +115,7 @@ typedef struct _VecI64Features { VecI64 (*alloc)(Py_ssize_t, Py_ssize_t); PyObject *(*box)(VecI64); VecI64 (*unbox)(PyObject *); + VecI64 (*convert_from_nested)(VecbufTExtItem); VecI64 (*append)(VecI64, int64_t); VecI64PopResult (*pop)(VecI64, Py_ssize_t); VecI64 (*remove)(VecI64, int64_t); @@ -146,6 +147,7 @@ typedef struct _VecTFeatures { VecT (*alloc)(Py_ssize_t, Py_ssize_t, size_t); PyObject *(*box)(VecT, size_t); VecT (*unbox)(PyObject *, size_t); + VecT (*convert_from_nested)(VecbufTExtItem); VecT (*append)(VecT, PyObject *, size_t); VecTPopResult (*pop)(VecT, Py_ssize_t); VecT (*remove)(VecT, PyObject *); @@ -178,6 +180,7 @@ typedef struct _VecTExtFeatures { VecTExt (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); PyObject *(*box)(VecTExt); VecTExt (*unbox)(PyObject *, size_t, size_t depth); + VecTExt (*convert_from_nested)(VecbufTExtItem); VecTExt (*append)(VecTExt, VecbufTExtItem); VecTExtPopResult (*pop)(VecTExt, Py_ssize_t); VecTExt (*remove)(VecTExt, VecbufTExtItem); From 285823cbf873d61b569ea7eb3b045b0613b1e64f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 14:47:09 +0100 Subject: [PATCH 104/208] Fix segfault --- mypyc/lib-rt/vecs/vec_t_ext.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 1d64f3be26753..50e535b2ab504 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -104,7 +104,7 @@ VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) { VecbufTExtItem item = vec.buf->items[start + i]; - Py_INCREF(item.buf); + Py_XINCREF(item.buf); res.buf->items[i] = item; } return res; From 83b236e5a156c10350edd0e6889ff033bc861f11 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 18:14:19 +0100 Subject: [PATCH 105/208] Support negative vec indexing --- mypyc/lib-rt/vecs/vec_i64.c | 4 ++++ mypyc/lib-rt/vecs/vec_t.c | 8 ++++++++ mypyc/lib-rt/vecs/vec_t_ext.c | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index d145eef4f7ba0..2b83651d22096 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -116,6 +116,8 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { VecI64 v = ((VecI64Object *)o)->vec; if ((size_t)i < (size_t)v.len) { return PyLong_FromLongLong(v.buf->items[i]); + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + return PyLong_FromLongLong(v.buf->items[i + v.len]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -153,6 +155,8 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { return NULL; if ((size_t)i < (size_t)vec.len) { return PyLong_FromLongLong(vec.buf->items[i]); + } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { + return PyLong_FromLongLong(vec.buf->items[i + vec.len]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index b20b2be7327c9..88b7becf4c3b3 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -92,6 +92,10 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { PyObject *item = v.buf->items[i]; Py_INCREF(item); return item; + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + PyObject *item = v.buf->items[i + v.len]; + Py_INCREF(item); + return item; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -134,6 +138,10 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { PyObject *item = vec.buf->items[i]; Py_INCREF(item); return item; + } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { + PyObject *item = vec.buf->items[i + vec.len]; + Py_INCREF(item); + return item; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 50e535b2ab504..b5350d87d01b2 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -78,6 +78,8 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { VecTExt v = ((VecTExtObject *)o)->vec; if ((size_t)i < (size_t)v.len) { return box_vec_item_by_index(v, i); + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + return box_vec_item_by_index(v, i + v.len); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -118,6 +120,8 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { return NULL; if ((size_t)i < (size_t)vec.len) { return box_vec_item_by_index(vec, i); + } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { + return box_vec_item_by_index(vec, i + vec.len); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; From fb1879f3ff2986f43fccebf8df5ad6a175a45d21 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sun, 3 Sep 2023 18:40:56 +0100 Subject: [PATCH 106/208] Support negative vec indexing in set item --- mypyc/lib-rt/vecs/vec_i64.c | 6 ++++++ mypyc/lib-rt/vecs/vec_t.c | 4 ++++ mypyc/lib-rt/vecs/vec_t_ext.c | 3 +++ 3 files changed, 13 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 2b83651d22096..db9231f498a22 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -193,6 +193,12 @@ int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { return -1; v.buf->items[i] = x; return 0; + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + long long x = PyLong_AsLongLong(o); + if (x == -1 && PyErr_Occurred()) + return -1; + v.buf->items[i + v.len] = x; + return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return -1; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 88b7becf4c3b3..c00965a6bc77d 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -178,6 +178,10 @@ int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { Py_INCREF(o); v.buf->items[i] = o; return 0; + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + Py_INCREF(o); + v.buf->items[i + v.len] = o; + return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return -1; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index b5350d87d01b2..baece1e261d18 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -152,6 +152,9 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTExt v = ((VecTExtObject *)self)->vec; + if ((size_t)i + (size_t)v.len < (size_t)v.len) { + i += v.len; + } if ((size_t)i < (size_t)v.len) { VecbufTExtItem item; if (Vec_T_Ext_UnboxItem(v, o, &item) < 0) From c3d39a28ae68a96fa5873e2af9b26590401806e2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 08:56:21 +0000 Subject: [PATCH 107/208] Replace Vec_T -> VecT --- mypyc/lib-rt/vecs/vec_t.c | 50 ++++++++++++++-------------- mypyc/lib-rt/vecs/vec_t_ext.c | 62 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vecs.c | 42 ++++++++++++------------ mypyc/lib-rt/vecs/vecs.h | 36 ++++++++++---------- 4 files changed, 95 insertions(+), 95 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index c00965a6bc77d..3b77d827224a4 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -30,12 +30,12 @@ static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { } else { buf = alloc_buf(size, item_type); if (buf == NULL) - return Vec_T_Error(); + return VecT_Error(); } return (VecT) { .buf = buf }; } -PyObject *Vec_T_Box(VecT vec, size_t item_type) { +PyObject *VecT_Box(VecT vec, size_t item_type) { // An unboxed empty vec may have a NULL buf, but a boxed vec must have it // allocated, since it contains the item type if (vec.buf == NULL) { @@ -51,7 +51,7 @@ PyObject *Vec_T_Box(VecT vec, size_t item_type) { return (PyObject *)obj; } -VecT Vec_T_Unbox(PyObject *obj, size_t item_type) { +VecT VecT_Unbox(PyObject *obj, size_t item_type) { if (obj->ob_type == &VecTType) { VecT result = ((VecTObject *)obj)->vec; if (result.buf->item_type == item_type) { @@ -61,14 +61,14 @@ VecT Vec_T_Unbox(PyObject *obj, size_t item_type) { } // TODO: Better error message, with name of type PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return Vec_T_Error(); + return VecT_Error(); } -VecT Vec_T_ConvertFromNested(VecbufTExtItem item) { +VecT VecT_ConvertFromNested(VecbufTExtItem item) { return (VecT) { item.len, (VecbufTObject *)item.buf }; } -VecT Vec_T_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { +VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { if (cap < size) cap = size; VecT vec = vec_t_alloc(cap, item_type); @@ -102,7 +102,7 @@ PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { } } -VecT Vec_T_Slice(VecT vec, int64_t start, int64_t end) { +VecT VecT_Slice(VecT vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -162,7 +162,7 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return Vec_T_Box(res, vec.buf->item_type); + return VecT_Box(res, vec.buf->item_type); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -209,7 +209,7 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type) { +VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { if (vec.buf == NULL) { VecT new = vec_t_alloc(1, item_type); if (VEC_IS_ERROR(new)) @@ -243,7 +243,7 @@ VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type) { } } -VecT Vec_T_Remove(VecT v, PyObject *arg) { +VecT VecT_Remove(VecT v, PyObject *arg) { PyObject **items = v.buf->items; for (Py_ssize_t i = 0; i < v.len; i++) { int match = 0; @@ -252,7 +252,7 @@ VecT Vec_T_Remove(VecT v, PyObject *arg) { else { int itemcmp = PyObject_RichCompareBool(items[i], arg, Py_EQ); if (itemcmp < 0) - return Vec_T_Error(); + return VecT_Error(); match = itemcmp; } if (match) { @@ -269,10 +269,10 @@ VecT Vec_T_Remove(VecT v, PyObject *arg) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return Vec_T_Error(); + return VecT_Error(); } -VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index) { +VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { VecTPopResult result; if (index < 0) @@ -280,7 +280,7 @@ VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = Vec_T_Error(); + result.f0 = VecT_Error(); result.f1 = NULL; return result; } @@ -412,7 +412,7 @@ PyTypeObject VecTType = { // TODO: free }; -PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { +PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable) { VecT v = vec_t_alloc(0, item_type); if (VEC_IS_ERROR(v)) return NULL; @@ -431,7 +431,7 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { Py_DECREF(item); return NULL; } - v = Vec_T_Append(v, item, item_type); + v = VecT_Append(v, item, item_type); Py_DECREF(item); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); @@ -444,18 +444,18 @@ PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable) { VEC_DECREF(v); return NULL; } - return Vec_T_Box(v, item_type); + return VecT_Box(v, item_type); } VecTFeatures TFeatures = { &VecTType, &VecbufTType, - Vec_T_New, - Vec_T_Box, - Vec_T_Unbox, - Vec_T_ConvertFromNested, - Vec_T_Append, - Vec_T_Pop, - Vec_T_Remove, - Vec_T_Slice, + VecT_New, + VecT_Box, + VecT_Unbox, + VecT_ConvertFromNested, + VecT_Append, + VecT_Pop, + VecT_Remove, + VecT_Slice, }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index baece1e261d18..9ef31a27e1df3 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -10,7 +10,7 @@ #include "vecs.h" static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { - return Vec_T_Ext_BoxItem(v, v.buf->items[index]); + return VecT_Ext_BoxItem(v, v.buf->items[index]); } @@ -19,7 +19,7 @@ static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); if (buf == NULL) - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); buf->item_type = item_type; buf->depth = depth; if (!vec_is_magic_item_type(item_type)) @@ -29,7 +29,7 @@ static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) return res; } -PyObject *Vec_T_Ext_Box(VecTExt vec) { +PyObject *VecT_Ext_Box(VecTExt vec) { VecTExtObject *obj = PyObject_GC_New(VecTExtObject, &VecTExtType); if (obj == NULL) return NULL; @@ -38,7 +38,7 @@ PyObject *Vec_T_Ext_Box(VecTExt vec) { return (PyObject *)obj; } -VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { +VecTExt VecT_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { if (obj->ob_type == &VecTExtType) { VecTExt result = ((VecTExtObject *)obj)->vec; if (result.buf->item_type == item_type && result.buf->depth == depth) { @@ -48,14 +48,14 @@ VecTExt Vec_T_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { } // TODO: Better error message, with name of type PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); } -VecTExt Vec_T_Ext_ConvertFromNested(VecbufTExtItem item) { +VecTExt VecT_Ext_ConvertFromNested(VecbufTExtItem item) { return (VecTExt) { item.len, (VecbufTExtObject *)item.buf }; } -VecTExt Vec_T_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { +VecTExt VecT_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; VecTExt vec = vec_t_ext_alloc(cap, item_type, depth); @@ -86,7 +86,7 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { } } -VecTExt Vec_T_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { +VecTExt VecT_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -142,7 +142,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return Vec_T_Ext_Box(res); + return VecT_Ext_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -157,7 +157,7 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } if ((size_t)i < (size_t)v.len) { VecbufTExtItem item; - if (Vec_T_Ext_UnboxItem(v, o, &item) < 0) + if (VecT_Ext_UnboxItem(v, o, &item) < 0) return -1; VEC_INCREF(item); VEC_DECREF(v.buf->items[i]); @@ -214,7 +214,7 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { } // Steals reference to vec (but not x) -VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { +VecTExt VecT_Ext_Append(VecTExt vec, VecbufTExtItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); if (vec.len < cap) { @@ -242,12 +242,12 @@ VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x) { } } -VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { +VecTExt VecT_Ext_Remove(VecTExt self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; - PyObject *boxed_arg = Vec_T_Ext_BoxItem(self, arg); + PyObject *boxed_arg = VecT_Ext_BoxItem(self, arg); if (boxed_arg == NULL) - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); for (Py_ssize_t i = 0; i < self.len; i++) { int match = 0; @@ -257,13 +257,13 @@ VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { PyObject *item = box_vec_item_by_index(self, i); if (item == NULL) { Py_DECREF(boxed_arg); - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); } int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); Py_DECREF(item); if (itemcmp < 0) { Py_DECREF(boxed_arg); - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); } match = itemcmp; } @@ -283,10 +283,10 @@ VecTExt Vec_T_Ext_Remove(VecTExt self, VecbufTExtItem arg) { } Py_DECREF(boxed_arg); PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return Vec_T_Ext_Error(); + return VecT_Ext_Error(); } -VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index) { +VecTExtPopResult VecT_Ext_Pop(VecTExt v, Py_ssize_t index) { VecTExtPopResult result; if (index < 0) @@ -294,7 +294,7 @@ VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = Vec_T_Ext_Error(); + result.f0 = VecT_Ext_Error(); result.f1.len = 0; result.f1.buf = NULL; return result; @@ -428,7 +428,7 @@ PyTypeObject VecTExtType = { // TODO: free }; -PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { +PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { VecTExt v = vec_t_ext_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; @@ -442,13 +442,13 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *itera PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { VecbufTExtItem vecitem; - if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) { + if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) { Py_DECREF(iter); VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = Vec_T_Ext_Append(v, vecitem); + v = VecT_Ext_Append(v, vecitem); Py_DECREF(item); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); @@ -461,18 +461,18 @@ PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *itera VEC_DECREF(v); return NULL; } - return Vec_T_Ext_Box(v); + return VecT_Ext_Box(v); } VecTExtFeatures TExtFeatures = { &VecTExtType, &VecbufTExtType, - Vec_T_Ext_New, - Vec_T_Ext_Box, - Vec_T_Ext_Unbox, - Vec_T_Ext_ConvertFromNested, - Vec_T_Ext_Append, - Vec_T_Ext_Pop, - Vec_T_Ext_Remove, - Vec_T_Ext_Slice, + VecT_Ext_New, + VecT_Ext_Box, + VecT_Ext_Unbox, + VecT_Ext_ConvertFromNested, + VecT_Ext_Append, + VecT_Ext_Pop, + VecT_Ext_Remove, + VecT_Ext_Slice, }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 63bc67c98790d..7dc9a184cc336 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -28,21 +28,21 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) VecProxy *p = (VecProxy *)self; if (p->depth == 0) { if (init == NULL) { - VecT vec = Vec_T_New(0, 0, p->item_type); + VecT vec = VecT_New(0, 0, p->item_type); if (VEC_IS_ERROR(vec)) return NULL; - return Vec_T_Box(vec, p->item_type); + return VecT_Box(vec, p->item_type); } else { - return Vec_T_FromIterable(p->item_type, init); + return VecT_FromIterable(p->item_type, init); } } else { if (init == NULL) { - VecTExt vec = Vec_T_Ext_New(0, 0, p->item_type, p->depth); + VecTExt vec = VecT_Ext_New(0, 0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; - return Vec_T_Ext_Box(vec); + return VecT_Ext_Box(vec); } else { - return Vec_T_Ext_FromIterable(p->item_type, p->depth, init); + return VecT_Ext_FromIterable(p->item_type, p->depth, init); } } } @@ -360,20 +360,20 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) return NULL; } VEC_INCREF(v); - v = Vec_T_Append(v, item, v.buf->item_type); + v = VecT_Append(v, item, v.buf->item_type); if (VEC_IS_ERROR(v)) return NULL; - return Vec_T_Box(v, v.buf->item_type); + return VecT_Box(v, v.buf->item_type); } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; - if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) + if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = Vec_T_Ext_Append(v, vecitem); + v = VecT_Ext_Append(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return Vec_T_Ext_Box(v); + return VecT_Ext_Box(v); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; @@ -407,20 +407,20 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) return NULL; } VEC_INCREF(v); - v = Vec_T_Remove(v, item); + v = VecT_Remove(v, item); if (VEC_IS_ERROR(v)) return NULL; - return Vec_T_Box(v, v.buf->item_type); + return VecT_Box(v, v.buf->item_type); } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; - if (Vec_T_Ext_UnboxItem(v, item, &vecitem) < 0) + if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = Vec_T_Ext_Remove(v, vecitem); + v = VecT_Ext_Remove(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return Vec_T_Ext_Box(v); + return VecT_Ext_Box(v); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; @@ -452,11 +452,11 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; - r = Vec_T_Pop(v, index); + r = VecT_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_T_Box(r.f0, v.buf->item_type); + result_item0 = VecT_Box(r.f0, v.buf->item_type); if (result_item0 == NULL) { Py_DECREF(r.f1); return NULL; @@ -465,18 +465,18 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecTExtPopResult r; - r = Vec_T_Ext_Pop(v, index); + r = VecT_Ext_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_T_Ext_Box(r.f0); + result_item0 = VecT_Ext_Box(r.f0); if (result_item0 == NULL) { Py_DECREF(r.f0.buf); Py_DECREF(r.f1.buf); return NULL; } - result_item1 = Vec_T_Ext_BoxItem(r.f0, r.f1); + result_item1 = VecT_Ext_BoxItem(r.f0, r.f1); if (result_item1 == NULL) { Py_DECREF(result_item0); Py_DECREF(r.f1.buf); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index ee0f278292057..86f3cf5de76eb 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -211,12 +211,12 @@ inline VecI64 Vec_I64_Error() { return v; } -inline VecT Vec_T_Error() { +inline VecT VecT_Error() { VecT v = { .len = -1 }; return v; } -inline VecTExt Vec_T_Ext_Error() { +inline VecTExt VecT_Ext_Error() { VecTExt v = { .len = -1 }; return v; } @@ -265,12 +265,12 @@ static inline int VecT_ItemCheck(VecT v, PyObject *item, size_t item_type) { } } -VecT Vec_T_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type); -PyObject *Vec_T_FromIterable(size_t item_type, PyObject *iterable); -PyObject *Vec_T_Box(VecT vec, size_t item_type); -VecT Vec_T_Append(VecT vec, PyObject *x, size_t item_type); -VecT Vec_T_Remove(VecT vec, PyObject *x); -VecTPopResult Vec_T_Pop(VecT v, Py_ssize_t index); +VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type); +PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable); +PyObject *VecT_Box(VecT vec, size_t item_type); +VecT VecT_Append(VecT vec, PyObject *x, size_t item_type); +VecT VecT_Remove(VecT vec, PyObject *x); +VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); // vec[t] operations (extended) @@ -296,16 +296,16 @@ static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { } } -VecTExt Vec_T_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); -PyObject *Vec_T_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable); -PyObject *Vec_T_Ext_Box(VecTExt); -VecTExt Vec_T_Ext_Append(VecTExt vec, VecbufTExtItem x); -VecTExt Vec_T_Ext_Remove(VecTExt vec, VecbufTExtItem x); -VecTExtPopResult Vec_T_Ext_Pop(VecTExt v, Py_ssize_t index); +VecTExt VecT_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); +PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable); +PyObject *VecT_Ext_Box(VecTExt); +VecTExt VecT_Ext_Append(VecTExt vec, VecbufTExtItem x); +VecTExt VecT_Ext_Remove(VecTExt vec, VecbufTExtItem x); +VecTExtPopResult VecT_Ext_Pop(VecTExt v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. -static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { +static inline int VecT_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { // TODO: vec[i64] @@ -336,14 +336,14 @@ static inline int Vec_T_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem return -1; } -static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { +static inline PyObject *VecT_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { if (item.len < 0) Py_RETURN_NONE; Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; - return Vec_T_Ext_Box(v); + return VecT_Ext_Box(v); } else { // Item is a non-nested vec size_t item_type = v.buf->item_type; @@ -354,7 +354,7 @@ static inline PyObject *Vec_T_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; - return Vec_T_Box(v, item_type); + return VecT_Box(v, item_type); } } } From f5d96dd284a999b2884d74e2146fb8a5eac13fd2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 08:57:42 +0000 Subject: [PATCH 108/208] Replace VecT_Ext -> VecVec --- mypyc/lib-rt/vecs/vec_t_ext.c | 62 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vecs.c | 24 +++++++------- mypyc/lib-rt/vecs/vecs.h | 20 +++++------ 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 9ef31a27e1df3..edf01cd496fdb 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -10,7 +10,7 @@ #include "vecs.h" static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { - return VecT_Ext_BoxItem(v, v.buf->items[index]); + return VecVec_BoxItem(v, v.buf->items[index]); } @@ -19,7 +19,7 @@ static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); if (buf == NULL) - return VecT_Ext_Error(); + return VecVec_Error(); buf->item_type = item_type; buf->depth = depth; if (!vec_is_magic_item_type(item_type)) @@ -29,7 +29,7 @@ static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) return res; } -PyObject *VecT_Ext_Box(VecTExt vec) { +PyObject *VecVec_Box(VecTExt vec) { VecTExtObject *obj = PyObject_GC_New(VecTExtObject, &VecTExtType); if (obj == NULL) return NULL; @@ -38,7 +38,7 @@ PyObject *VecT_Ext_Box(VecTExt vec) { return (PyObject *)obj; } -VecTExt VecT_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { +VecTExt VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { if (obj->ob_type == &VecTExtType) { VecTExt result = ((VecTExtObject *)obj)->vec; if (result.buf->item_type == item_type && result.buf->depth == depth) { @@ -48,14 +48,14 @@ VecTExt VecT_Ext_Unbox(PyObject *obj, size_t item_type, size_t depth) { } // TODO: Better error message, with name of type PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return VecT_Ext_Error(); + return VecVec_Error(); } -VecTExt VecT_Ext_ConvertFromNested(VecbufTExtItem item) { +VecTExt VecVec_ConvertFromNested(VecbufTExtItem item) { return (VecTExt) { item.len, (VecbufTExtObject *)item.buf }; } -VecTExt VecT_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { +VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; VecTExt vec = vec_t_ext_alloc(cap, item_type, depth); @@ -86,7 +86,7 @@ PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { } } -VecTExt VecT_Ext_Slice(VecTExt vec, int64_t start, int64_t end) { +VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -142,7 +142,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return VecT_Ext_Box(res); + return VecVec_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -157,7 +157,7 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } if ((size_t)i < (size_t)v.len) { VecbufTExtItem item; - if (VecT_Ext_UnboxItem(v, o, &item) < 0) + if (VecVec_UnboxItem(v, o, &item) < 0) return -1; VEC_INCREF(item); VEC_DECREF(v.buf->items[i]); @@ -214,7 +214,7 @@ PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { } // Steals reference to vec (but not x) -VecTExt VecT_Ext_Append(VecTExt vec, VecbufTExtItem x) { +VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); if (vec.len < cap) { @@ -242,12 +242,12 @@ VecTExt VecT_Ext_Append(VecTExt vec, VecbufTExtItem x) { } } -VecTExt VecT_Ext_Remove(VecTExt self, VecbufTExtItem arg) { +VecTExt VecVec_Remove(VecTExt self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; - PyObject *boxed_arg = VecT_Ext_BoxItem(self, arg); + PyObject *boxed_arg = VecVec_BoxItem(self, arg); if (boxed_arg == NULL) - return VecT_Ext_Error(); + return VecVec_Error(); for (Py_ssize_t i = 0; i < self.len; i++) { int match = 0; @@ -257,13 +257,13 @@ VecTExt VecT_Ext_Remove(VecTExt self, VecbufTExtItem arg) { PyObject *item = box_vec_item_by_index(self, i); if (item == NULL) { Py_DECREF(boxed_arg); - return VecT_Ext_Error(); + return VecVec_Error(); } int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); Py_DECREF(item); if (itemcmp < 0) { Py_DECREF(boxed_arg); - return VecT_Ext_Error(); + return VecVec_Error(); } match = itemcmp; } @@ -283,10 +283,10 @@ VecTExt VecT_Ext_Remove(VecTExt self, VecbufTExtItem arg) { } Py_DECREF(boxed_arg); PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return VecT_Ext_Error(); + return VecVec_Error(); } -VecTExtPopResult VecT_Ext_Pop(VecTExt v, Py_ssize_t index) { +VecTExtPopResult VecVec_Pop(VecTExt v, Py_ssize_t index) { VecTExtPopResult result; if (index < 0) @@ -294,7 +294,7 @@ VecTExtPopResult VecT_Ext_Pop(VecTExt v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = VecT_Ext_Error(); + result.f0 = VecVec_Error(); result.f1.len = 0; result.f1.buf = NULL; return result; @@ -428,7 +428,7 @@ PyTypeObject VecTExtType = { // TODO: free }; -PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { +PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { VecTExt v = vec_t_ext_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; @@ -442,13 +442,13 @@ PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterab PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { VecbufTExtItem vecitem; - if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) { + if (VecVec_UnboxItem(v, item, &vecitem) < 0) { Py_DECREF(iter); VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = VecT_Ext_Append(v, vecitem); + v = VecVec_Append(v, vecitem); Py_DECREF(item); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); @@ -461,18 +461,18 @@ PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterab VEC_DECREF(v); return NULL; } - return VecT_Ext_Box(v); + return VecVec_Box(v); } VecTExtFeatures TExtFeatures = { &VecTExtType, &VecbufTExtType, - VecT_Ext_New, - VecT_Ext_Box, - VecT_Ext_Unbox, - VecT_Ext_ConvertFromNested, - VecT_Ext_Append, - VecT_Ext_Pop, - VecT_Ext_Remove, - VecT_Ext_Slice, + VecVec_New, + VecVec_Box, + VecVec_Unbox, + VecVec_ConvertFromNested, + VecVec_Append, + VecVec_Pop, + VecVec_Remove, + VecVec_Slice, }; diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 7dc9a184cc336..943c1497c23b2 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -37,12 +37,12 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } } else { if (init == NULL) { - VecTExt vec = VecT_Ext_New(0, 0, p->item_type, p->depth); + VecTExt vec = VecVec_New(0, 0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; - return VecT_Ext_Box(vec); + return VecVec_Box(vec); } else { - return VecT_Ext_FromIterable(p->item_type, p->depth, init); + return VecVec_FromIterable(p->item_type, p->depth, init); } } } @@ -367,13 +367,13 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; - if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) + if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = VecT_Ext_Append(v, vecitem); + v = VecVec_Append(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return VecT_Ext_Box(v); + return VecVec_Box(v); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; @@ -414,13 +414,13 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; - if (VecT_Ext_UnboxItem(v, item, &vecitem) < 0) + if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = VecT_Ext_Remove(v, vecitem); + v = VecVec_Remove(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return VecT_Ext_Box(v); + return VecVec_Box(v); } else { PyErr_SetString(PyExc_TypeError, "vec argument expected"); return NULL; @@ -465,18 +465,18 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) } else if (VecTExt_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecTExtPopResult r; - r = VecT_Ext_Pop(v, index); + r = VecVec_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = VecT_Ext_Box(r.f0); + result_item0 = VecVec_Box(r.f0); if (result_item0 == NULL) { Py_DECREF(r.f0.buf); Py_DECREF(r.f1.buf); return NULL; } - result_item1 = VecT_Ext_BoxItem(r.f0, r.f1); + result_item1 = VecVec_BoxItem(r.f0, r.f1); if (result_item1 == NULL) { Py_DECREF(result_item0); Py_DECREF(r.f1.buf); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 86f3cf5de76eb..78299155f2bb3 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -216,7 +216,7 @@ inline VecT VecT_Error() { return v; } -inline VecTExt VecT_Ext_Error() { +inline VecTExt VecVec_Error() { VecTExt v = { .len = -1 }; return v; } @@ -296,16 +296,16 @@ static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { } } -VecTExt VecT_Ext_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); -PyObject *VecT_Ext_FromIterable(size_t item_type, size_t depth, PyObject *iterable); -PyObject *VecT_Ext_Box(VecTExt); -VecTExt VecT_Ext_Append(VecTExt vec, VecbufTExtItem x); -VecTExt VecT_Ext_Remove(VecTExt vec, VecbufTExtItem x); -VecTExtPopResult VecT_Ext_Pop(VecTExt v, Py_ssize_t index); +VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); +PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable); +PyObject *VecVec_Box(VecTExt); +VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x); +VecTExt VecVec_Remove(VecTExt vec, VecbufTExtItem x); +VecTExtPopResult VecVec_Pop(VecTExt v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. -static inline int VecT_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { +static inline int VecVec_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { // TODO: vec[i64] @@ -336,14 +336,14 @@ static inline int VecT_Ext_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem * return -1; } -static inline PyObject *VecT_Ext_BoxItem(VecTExt v, VecbufTExtItem item) { +static inline PyObject *VecVec_BoxItem(VecTExt v, VecbufTExtItem item) { if (item.len < 0) Py_RETURN_NONE; Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; - return VecT_Ext_Box(v); + return VecVec_Box(v); } else { // Item is a non-nested vec size_t item_type = v.buf->item_type; From bcba2da91bebc3f6bf8b623a9bf08e708f221156 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 08:59:32 +0000 Subject: [PATCH 109/208] Replace Vec_I64 -> VecI64 --- mypyc/lib-rt/vecs/vec_i64.c | 54 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vec_t_ext.c | 6 ++-- mypyc/lib-rt/vecs/vecs.c | 12 ++++---- mypyc/lib-rt/vecs/vecs.h | 12 ++++---- 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index db9231f498a22..ae06096cefd49 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -12,7 +12,7 @@ static VecI64 vec_i64_alloc(Py_ssize_t size) } else { buf = PyObject_NewVar(VecbufI64Object, &VecbufI64Type, size); if (buf == NULL) - return Vec_I64_Error(); + return VecI64_Error(); } VecI64 res = { .buf = buf }; return res; @@ -23,7 +23,7 @@ static void vec_i64_dealloc(VecI64Object *self) { PyObject_Del(self); } -PyObject *Vec_I64_Box(VecI64 vec) { +PyObject *VecI64_Box(VecI64 vec) { VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); if (obj == NULL) return NULL; @@ -31,7 +31,7 @@ PyObject *Vec_I64_Box(VecI64 vec) { return (PyObject *)obj; } -VecI64 Vec_I64_Unbox(PyObject *obj) { +VecI64 VecI64_Unbox(PyObject *obj) { if (obj->ob_type == &VecI64Type) { VecI64 result = ((VecI64Object *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? @@ -39,15 +39,15 @@ VecI64 Vec_I64_Unbox(PyObject *obj) { } else { // TODO: Better error message PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); - return Vec_I64_Error(); + return VecI64_Error(); } } -VecI64 Vec_I64_ConvertFromNested(VecbufTExtItem item) { +VecI64 VecI64_ConvertFromNested(VecbufTExtItem item) { return (VecI64) { item.len, (VecbufI64Object *)item.buf }; } -VecI64 Vec_I64_New(Py_ssize_t size, Py_ssize_t cap) { +VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; VecI64 vec = vec_i64_alloc(cap); @@ -60,7 +60,7 @@ VecI64 Vec_I64_New(Py_ssize_t size, Py_ssize_t cap) { return vec; } -PyObject *Vec_I64_FromIterable(PyObject *iterable) { +PyObject *VecI64_FromIterable(PyObject *iterable) { VecI64 v = vec_i64_alloc(0); if (VEC_IS_ERROR(v)) return NULL; @@ -80,7 +80,7 @@ PyObject *Vec_I64_FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - v = Vec_I64_Append(v, x); + v = VecI64_Append(v, x); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); VEC_DECREF(v); @@ -92,7 +92,7 @@ PyObject *Vec_I64_FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - return Vec_I64_Box(v); + return VecI64_Box(v); } PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { @@ -102,9 +102,9 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return NULL; } if (init == NULL) { - return Vec_I64_Box(Vec_I64_New(0, 0)); + return VecI64_Box(VecI64_New(0, 0)); } else { - return (PyObject *)Vec_I64_FromIterable(init); + return (PyObject *)VecI64_FromIterable(init); } } @@ -124,7 +124,7 @@ PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { } } -VecI64 Vec_I64_Slice(VecI64 vec, int64_t start, int64_t end) { +VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -175,7 +175,7 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = vec.buf->items[j]; j += step; } - return Vec_I64_Box(res); + return VecI64_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -239,7 +239,7 @@ static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { +VecI64 VecI64_Append(VecI64 vec, int64_t x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; @@ -249,7 +249,7 @@ VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { Py_ssize_t new_size = 2 * cap + 1; VecI64 new = vec_i64_alloc(new_size); if (VEC_IS_ERROR(new)) - return Vec_I64_Error(); + return VecI64_Error(); new.len = vec.len + 1; if (vec.len > 0) memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); @@ -259,7 +259,7 @@ VecI64 Vec_I64_Append(VecI64 vec, int64_t x) { } } -VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { +VecI64 VecI64_Remove(VecI64 v, int64_t x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -271,10 +271,10 @@ VecI64 Vec_I64_Remove(VecI64 v, int64_t x) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return Vec_I64_Error(); + return VecI64_Error(); } -VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index) { +VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { VecI64PopResult result; if (index < 0) @@ -282,7 +282,7 @@ VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = Vec_I64_Error(); + result.f0 = VecI64_Error(); result.f1 = 0; return result; } @@ -343,12 +343,12 @@ PyTypeObject VecI64Type = { VecI64Features I64Features = { &VecI64Type, &VecbufI64Type, - Vec_I64_New, - Vec_I64_Box, - Vec_I64_Unbox, - Vec_I64_ConvertFromNested, - Vec_I64_Append, - Vec_I64_Pop, - Vec_I64_Remove, - Vec_I64_Slice, + VecI64_New, + VecI64_Box, + VecI64_Unbox, + VecI64_ConvertFromNested, + VecI64_Append, + VecI64_Pop, + VecI64_Remove, + VecI64_Slice, }; diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index edf01cd496fdb..da39350b03252 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -1,9 +1,9 @@ -// Implementation of generic vec[t], when t is an optional type or nested generic vec. +// Implementation of nested vec[t], when t is a vec type. // // Examples of types supported: -// - vec[str | None] +// - vec[vec[i64]] // - vec[vec[str]] -// - vec[vec[str | None] | None] +// - vec[vec[str | None]] #define PY_SSIZE_T_CLEAN #include diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 943c1497c23b2..5a82c5e08348b 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -350,10 +350,10 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) } VecI64 v = ((VecI64Object *)vec)->vec; VEC_INCREF(v); - v = Vec_I64_Append(v, x); + v = VecI64_Append(v, x); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return Vec_I64_Box(v); + return VecI64_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -397,10 +397,10 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) } VecI64 v = ((VecI64Object *)vec)->vec; VEC_INCREF(v); - v = Vec_I64_Remove(v, x); + v = VecI64_Remove(v, x); if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? - return Vec_I64_Box(v); + return VecI64_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -441,11 +441,11 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) if (VecI64_Check(vec)) { VecI64 v = ((VecI64Object *)vec)->vec; VecI64PopResult r; - r = Vec_I64_Pop(v, index); + r = VecI64_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = Vec_I64_Box(r.f0); + result_item0 = VecI64_Box(r.f0); if (result_item0 == NULL) return NULL; result_item1 = PyLong_FromLongLong(r.f1); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 78299155f2bb3..c11dfb91be44f 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -206,7 +206,7 @@ typedef struct { #define VEC_DECREF(v) Py_XDECREF((v).buf) #define VEC_INCREF(v) Py_XINCREF((v).buf) -inline VecI64 Vec_I64_Error() { +inline VecI64 VecI64_Error() { VecI64 v = { .len = -1 }; return v; } @@ -242,10 +242,10 @@ static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } -PyObject *Vec_I64_Box(VecI64); -VecI64 Vec_I64_Append(VecI64, int64_t x); -VecI64 Vec_I64_Remove(VecI64, int64_t x); -VecI64PopResult Vec_I64_Pop(VecI64 v, Py_ssize_t index); +PyObject *VecI64_Box(VecI64); +VecI64 VecI64_Append(VecI64, int64_t x); +VecI64 VecI64_Remove(VecI64, int64_t x); +VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); // vec[t] operations (simple) @@ -350,7 +350,7 @@ static inline PyObject *VecVec_BoxItem(VecTExt v, VecbufTExtItem item) { if (item_type == VEC_ITEM_TYPE_I64) { // vec[i64] VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; - return Vec_I64_Box(v); + return VecI64_Box(v); } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; From 5b4344ba17eaaff386f99de1095f48cdc73211f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:04:39 +0000 Subject: [PATCH 110/208] More refactoring --- mypyc/lib-rt/vecs/vec_i64.c | 46 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_t_ext.c | 2 +- mypyc/lib-rt/vecs/vecs.c | 4 +-- mypyc/lib-rt/vecs/vecs.h | 2 +- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index ae06096cefd49..ad42d5a4bbf55 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -3,7 +3,7 @@ #include "vecs.h" // Alloc a partially initialized vec. Caller *must* initialize len. -static VecI64 vec_i64_alloc(Py_ssize_t size) +static VecI64 vec_alloc(Py_ssize_t size) { VecbufI64Object *buf; /* TODO: Check for overflow */ @@ -18,7 +18,7 @@ static VecI64 vec_i64_alloc(Py_ssize_t size) return res; } -static void vec_i64_dealloc(VecI64Object *self) { +static void vec_dealloc(VecI64Object *self) { Py_CLEAR(self->vec.buf); PyObject_Del(self); } @@ -50,7 +50,7 @@ VecI64 VecI64_ConvertFromNested(VecbufTExtItem item) { VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; - VecI64 vec = vec_i64_alloc(cap); + VecI64 vec = vec_alloc(cap); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < cap; i++) { @@ -61,7 +61,7 @@ VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { } PyObject *VecI64_FromIterable(PyObject *iterable) { - VecI64 v = vec_i64_alloc(0); + VecI64 v = vec_alloc(0); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; @@ -95,7 +95,7 @@ PyObject *VecI64_FromIterable(PyObject *iterable) { return VecI64_Box(v); } -PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { +PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { static char *kwlist[] = {"", NULL}; PyObject *init = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { @@ -108,11 +108,11 @@ PyObject *vec_i64_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } } -PyObject *vec_i64_repr(PyObject *self) { - return vec_repr(self, (size_t)I64TypeObj, 0, 1); +PyObject *vec_repr(PyObject *self) { + return Vec_GenericRepr(self, (size_t)I64TypeObj, 0, 1); } -PyObject *vec_i64_get_item(PyObject *o, Py_ssize_t i) { +PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { VecI64 v = ((VecI64Object *)o)->vec; if ((size_t)i < (size_t)v.len) { return PyLong_FromLongLong(v.buf->items[i]); @@ -138,7 +138,7 @@ VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecI64 res = vec_i64_alloc(slicelength); + VecI64 res = vec_alloc(slicelength); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -147,7 +147,7 @@ VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { return res; } -PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { +PyObject *vec_subscript(PyObject *self, PyObject *item) { VecI64 vec = ((VecI64Object *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -166,7 +166,7 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecI64 res = vec_i64_alloc(slicelength); + VecI64 res = vec_alloc(slicelength); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -183,7 +183,7 @@ PyObject *vec_i64_subscript(PyObject *self, PyObject *item) { } } -int vec_i64_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { +int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (check_float_error(o)) return -1; VecI64 v = ((VecI64Object *)self)->vec; @@ -210,7 +210,7 @@ static Py_ssize_t vec_length(PyObject *o) { return ((VecI64Object *)o)->vec.len; } -static PyObject *vec_i64_richcompare(PyObject *self, PyObject *other, int op) { +static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { int cmp = 1; PyObject *res; if (op == Py_EQ || op == Py_NE) { @@ -247,7 +247,7 @@ VecI64 VecI64_Append(VecI64 vec, int64_t x) { } else { Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; Py_ssize_t new_size = 2 * cap + 1; - VecI64 new = vec_i64_alloc(new_size); + VecI64 new = vec_alloc(new_size); if (VEC_IS_ERROR(new)) return VecI64_Error(); new.len = vec.len + 1; @@ -300,15 +300,15 @@ VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { static PyMappingMethods VecI64Mapping = { .mp_length = vec_length, - .mp_subscript = vec_i64_subscript, + .mp_subscript = vec_subscript, }; static PySequenceMethods VecI64Sequence = { - .sq_item = vec_i64_get_item, - .sq_ass_item = vec_i64_ass_item, + .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, }; -static PyMethodDef vec_i64_methods[] = { +static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; @@ -330,14 +330,14 @@ PyTypeObject VecI64Type = { .tp_basicsize = sizeof(VecI64Object), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = vec_i64_new, + .tp_new = vec_new, //.tp_free = PyObject_Del, - .tp_dealloc = (destructor)vec_i64_dealloc, - .tp_repr = (reprfunc)vec_i64_repr, + .tp_dealloc = (destructor)vec_dealloc, + .tp_repr = (reprfunc)vec_repr, .tp_as_sequence = &VecI64Sequence, .tp_as_mapping = &VecI64Mapping, - .tp_richcompare = vec_i64_richcompare, - .tp_methods = vec_i64_methods, + .tp_richcompare = vec_richcompare, + .tp_methods = vec_methods, }; VecI64Features I64Features = { diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3b77d827224a4..3a9b93e279b34 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -83,7 +83,7 @@ VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { PyObject *vec_t_repr(PyObject *self) { VecTObject *v = (VecTObject *)self; - return vec_repr(self, v->vec.buf->item_type, 0, 1); + return Vec_GenericRepr(self, v->vec.buf->item_type, 0, 1); } PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index da39350b03252..75315202890bd 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -71,7 +71,7 @@ VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t dep PyObject *vec_t_ext_repr(PyObject *self) { VecTExt v = ((VecTExtObject *)self)->vec; - return vec_repr(self, v.buf->item_type, v.buf->depth, 1); + return Vec_GenericRepr(self, v.buf->item_type, v.buf->depth, 1); } PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 5a82c5e08348b..b53fbac73496d 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -210,7 +210,7 @@ PyTypeObject VecGenericType = { .tp_methods = vec_methods, }; -PyObject *vec_repr(PyObject *vec, size_t item_type, size_t depth, int verbose) { +PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose) { // TODO: Check for errors // TODO: Refcount handling PyObject *l = Py_BuildValue("[]"); @@ -239,7 +239,7 @@ PyObject *vec_repr(PyObject *vec, size_t item_type, size_t depth, int verbose) { if (depth == 0 || it == Py_None) { r = PyObject_Repr(it); } else { - r = vec_repr(it, item_type, depth - 1, 0); + r = Vec_GenericRepr(it, item_type, depth - 1, 0); } if (r == NULL) return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index c11dfb91be44f..9d35f97f4e922 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -370,7 +370,7 @@ static inline int check_float_error(PyObject *o) { } PyObject *vec_type_to_str(size_t item_type, size_t depth); -PyObject *vec_repr(PyObject *vec, size_t item_type, size_t depth, int verbose); +PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose); PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); From 1b5f8153033b14f9471be7d2916f4a99e10e01ca Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:05:27 +0000 Subject: [PATCH 111/208] More renaming --- mypyc/lib-rt/vecs/vec_t_ext.c | 40 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 75315202890bd..64a47349970a5 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -16,7 +16,7 @@ static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { // Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the // return value. -static VecTExt vec_t_ext_alloc(Py_ssize_t size, size_t item_type, size_t depth) { +static VecTExt vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); if (buf == NULL) return VecVec_Error(); @@ -58,7 +58,7 @@ VecTExt VecVec_ConvertFromNested(VecbufTExtItem item) { VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; - VecTExt vec = vec_t_ext_alloc(cap, item_type, depth); + VecTExt vec = vec_alloc(cap, item_type, depth); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < cap; i++) { @@ -69,12 +69,12 @@ VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t dep return vec; } -PyObject *vec_t_ext_repr(PyObject *self) { +static PyObject *vec_repr(PyObject *self) { VecTExt v = ((VecTExtObject *)self)->vec; return Vec_GenericRepr(self, v.buf->item_type, v.buf->depth, 1); } -PyObject *vec_t_ext_get_item(PyObject *o, Py_ssize_t i) { +static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { VecTExt v = ((VecTExtObject *)o)->vec; if ((size_t)i < (size_t)v.len) { return box_vec_item_by_index(v, i); @@ -100,7 +100,7 @@ VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->depth); + VecTExt res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -112,7 +112,7 @@ VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { return res; } -PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { +static PyObject *vec_subscript(PyObject *self, PyObject *item) { VecTExt vec = ((VecTExtObject *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -131,7 +131,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecTExt res = vec_t_ext_alloc(slicelength, vec.buf->item_type, vec.buf->depth); + VecTExt res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -150,7 +150,7 @@ PyObject *vec_t_ext_subscript(PyObject *self, PyObject *item) { } } -int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { +static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecTExt v = ((VecTExtObject *)self)->vec; if ((size_t)i + (size_t)v.len < (size_t)v.len) { i += v.len; @@ -169,7 +169,7 @@ int vec_t_ext_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -static PyObject *compare_vec_t_ext_eq(VecTExt x, VecTExt y, int op) { +static PyObject *compare_vec_eq(VecTExt x, VecTExt y, int op) { int cmp = 1; PyObject *res; if (x.len != y.len @@ -198,13 +198,13 @@ static PyObject *compare_vec_t_ext_eq(VecTExt x, VecTExt y, int op) { return res; } -PyObject *vec_t_ext_richcompare(PyObject *self, PyObject *other, int op) { +PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { PyObject *res; if (op == Py_EQ || op == Py_NE) { if (other->ob_type != &VecTExtType) { res = op == Py_EQ ? Py_False : Py_True; } else { - return compare_vec_t_ext_eq(((VecTExtObject *)self)->vec, + return compare_vec_eq(((VecTExtObject *)self)->vec, ((VecTExtObject *)other)->vec, op); } } else @@ -224,7 +224,7 @@ VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTExt new = vec_t_ext_alloc(new_size, vec.buf->item_type, vec.buf->depth); + VecTExt new = vec_alloc(new_size, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(new)) { VEC_DECREF(x); return new; @@ -383,15 +383,15 @@ static Py_ssize_t vec_ext_length(PyObject *o) { static PyMappingMethods VecTExtMapping = { .mp_length = vec_ext_length, - .mp_subscript = vec_t_ext_subscript, + .mp_subscript = vec_subscript, }; static PySequenceMethods VecTExtSequence = { - .sq_item = vec_t_ext_get_item, - .sq_ass_item = vec_t_ext_ass_item, + .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, }; -static PyMethodDef vec_t_ext_methods[] = { +static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; @@ -420,16 +420,16 @@ PyTypeObject VecTExtType = { .tp_clear = (inquiry)VecTExt_clear, .tp_dealloc = (destructor)VecTExt_dealloc, //.tp_free = PyObject_GC_Del, - .tp_repr = (reprfunc)vec_t_ext_repr, + .tp_repr = (reprfunc)vec_repr, .tp_as_sequence = &VecTExtSequence, .tp_as_mapping = &VecTExtMapping, - .tp_richcompare = vec_t_ext_richcompare, - .tp_methods = vec_t_ext_methods, + .tp_richcompare = vec_richcompare, + .tp_methods = vec_methods, // TODO: free }; PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { - VecTExt v = vec_t_ext_alloc(0, item_type, depth); + VecTExt v = vec_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; From b01756c02ce286617f0bd49d6feeef0e43bbc3dc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:07:30 +0000 Subject: [PATCH 112/208] More refactoring --- mypyc/lib-rt/vecs/vec_t.c | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3a9b93e279b34..3ffdc35a8aa9e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -22,7 +22,7 @@ static inline VecbufTObject *alloc_buf(Py_ssize_t size, size_t item_type) { // Alloc a partially initialized vec. Caller *must* immediately initialize len (and buf->items // if size > 0). -static VecT vec_t_alloc(Py_ssize_t size, size_t item_type) { +static VecT vec_alloc(Py_ssize_t size, size_t item_type) { VecbufTObject *buf; if (size == 0) { @@ -71,7 +71,7 @@ VecT VecT_ConvertFromNested(VecbufTExtItem item) { VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { if (cap < size) cap = size; - VecT vec = vec_t_alloc(cap, item_type); + VecT vec = vec_alloc(cap, item_type); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < cap; i++) { @@ -81,12 +81,12 @@ VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { return vec; } -PyObject *vec_t_repr(PyObject *self) { +static PyObject *vec_repr(PyObject *self) { VecTObject *v = (VecTObject *)self; return Vec_GenericRepr(self, v->vec.buf->item_type, 0, 1); } -PyObject *vec_t_get_item(PyObject *o, Py_ssize_t i) { +static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { VecT v = ((VecTObject *)o)->vec; if ((size_t)i < (size_t)v.len) { PyObject *item = v.buf->items[i]; @@ -116,7 +116,7 @@ VecT VecT_Slice(VecT vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecT res = vec_t_alloc(slicelength, vec.buf->item_type); + VecT res = vec_alloc(slicelength, vec.buf->item_type); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -128,7 +128,7 @@ VecT VecT_Slice(VecT vec, int64_t start, int64_t end) { return res; } -PyObject *vec_t_subscript(PyObject *self, PyObject *item) { +static PyObject *vec_subscript(PyObject *self, PyObject *item) { VecT vec = ((VecTObject *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -151,7 +151,7 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecT res = vec_t_alloc(slicelength, vec.buf->item_type); + VecT res = vec_alloc(slicelength, vec.buf->item_type); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -170,7 +170,7 @@ PyObject *vec_t_subscript(PyObject *self, PyObject *item) { } } -int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { +static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { VecT v = ((VecTObject *)self)->vec; if (!VecT_ItemCheck(v, o, v.buf->item_type)) return -1; @@ -188,7 +188,7 @@ int vec_t_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { +static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { PyObject *res; if (op == Py_EQ || op == Py_NE) { if (other->ob_type != &VecTType) { @@ -211,7 +211,7 @@ PyObject *vec_t_richcompare(PyObject *self, PyObject *other, int op) { VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { if (vec.buf == NULL) { - VecT new = vec_t_alloc(1, item_type); + VecT new = vec_alloc(1, item_type); if (VEC_IS_ERROR(new)) return new; Py_INCREF(x); @@ -228,7 +228,7 @@ VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecT new = vec_t_alloc(new_size, vec.buf->item_type); + VecT new = vec_alloc(new_size, vec.buf->item_type); if (VEC_IS_ERROR(new)) return new; // Copy items to new vec. @@ -367,15 +367,15 @@ static Py_ssize_t vec_length(PyObject *o) { static PyMappingMethods VecTMapping = { .mp_length = vec_length, - .mp_subscript = vec_t_subscript, + .mp_subscript = vec_subscript, }; static PySequenceMethods VecTSequence = { - .sq_item = vec_t_get_item, - .sq_ass_item = vec_t_ass_item, + .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, }; -static PyMethodDef vec_t_methods[] = { +static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; @@ -404,16 +404,16 @@ PyTypeObject VecTType = { .tp_clear = (inquiry)VecT_clear, .tp_dealloc = (destructor)VecT_dealloc, //.tp_free = PyObject_GC_Del, - .tp_repr = (reprfunc)vec_t_repr, + .tp_repr = (reprfunc)vec_repr, .tp_as_sequence = &VecTSequence, .tp_as_mapping = &VecTMapping, - .tp_richcompare = vec_t_richcompare, - .tp_methods = vec_t_methods, + .tp_richcompare = vec_richcompare, + .tp_methods = vec_methods, // TODO: free }; PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable) { - VecT v = vec_t_alloc(0, item_type); + VecT v = vec_alloc(0, item_type); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; From 95c7b3a8d6301aa22691425c1ddda79e992a28db Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:08:33 +0000 Subject: [PATCH 113/208] More refactoring --- mypyc/lib-rt/vecs/vecs.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index b53fbac73496d..ef04ea9370be2 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -333,7 +333,7 @@ PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *a // Module-level functions -static PyObject *vecs_append(PyObject *self, PyObject *args) +static PyObject *vec_append(PyObject *self, PyObject *args) { PyObject *vec; PyObject *item; @@ -380,7 +380,7 @@ static PyObject *vecs_append(PyObject *self, PyObject *args) } } -static PyObject *vecs_remove(PyObject *self, PyObject *args) +static PyObject *vec_remove(PyObject *self, PyObject *args) { PyObject *vec; PyObject *item; @@ -427,7 +427,7 @@ static PyObject *vecs_remove(PyObject *self, PyObject *args) } } -static PyObject *vecs_pop(PyObject *self, PyObject *args) +static PyObject *vec_pop(PyObject *self, PyObject *args) { PyObject *vec; Py_ssize_t index = -1; @@ -505,9 +505,9 @@ static PyObject *vecs_pop(PyObject *self, PyObject *args) } static PyMethodDef VecsMethods[] = { - {"append", vecs_append, METH_VARARGS, "Append a value to the end of a vec"}, - {"remove", vecs_remove, METH_VARARGS, "Remove first occurrence of value"}, - {"pop", vecs_pop, METH_VARARGS, "Remove and return item at index (default last)"}, + {"append", vec_append, METH_VARARGS, "Append a value to the end of a vec"}, + {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value"}, + {"pop", vec_pop, METH_VARARGS, "Remove and return item at index (default last)"}, {NULL, NULL, 0, NULL} /* Sentinel */ }; From 44b9afd87ffe2dd6600d3acb69cffbf05af5e30d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:10:13 +0000 Subject: [PATCH 114/208] More refactoring --- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vecs.c | 18 +++++++++--------- mypyc/lib-rt/vecs/vecs.h | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 3ffdc35a8aa9e..b71ef52cdaf57 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -200,7 +200,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { res = op == Py_EQ ? Py_False : Py_True; } else { // TODO: why pointers to len? - return vec_generic_richcompare(&x.len, x.buf->items, &y.len, y.buf->items, op); + return Vec_GenericRichcompare(&x.len, x.buf->items, &y.len, y.buf->items, op); } } } else diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index ef04ea9370be2..a996248d744f4 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -65,7 +65,7 @@ VecProxy_dealloc(VecProxy *self) PyObject_GC_Del(self); } -PyObject *vec_type_to_str(size_t item_type, size_t depth) { +PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { PyObject *item; if (depth == 0) if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { @@ -77,7 +77,7 @@ PyObject *vec_type_to_str(size_t item_type, size_t depth) { } } else - item = vec_type_to_str(item_type, depth - 1); + item = Vec_TypeToStr(item_type, depth - 1); return PyUnicode_FromFormat("vec[%U]", item); } @@ -90,7 +90,7 @@ PyObject *VecProxy_repr(PyObject *self) { PyObject *sep = Py_BuildValue("s", ""); PyList_Append(l, prefix); VecProxy *v = (VecProxy *)self; - PyList_Append(l, vec_type_to_str(v->item_type, v->depth)); + PyList_Append(l, Vec_TypeToStr(v->item_type, v->depth)); PyList_Append(l, suffix); return PyUnicode_Join(sep, l); } @@ -220,7 +220,7 @@ PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int ver PyObject *sep = Py_BuildValue("s", ""); PyObject *comma = Py_BuildValue("s", ", "); if (verbose) { - prefix = vec_type_to_str(item_type, depth); + prefix = Vec_TypeToStr(item_type, depth); mid = Py_BuildValue("s", "(["); suffix = Py_BuildValue("s", "])"); } else { @@ -254,7 +254,7 @@ PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int ver } // Generic comparison implementation for vecs with PyObject * items. -PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, +PyObject *Vec_GenericRichcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op) { int cmp = 1; @@ -282,7 +282,7 @@ PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, return res; } -int vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) { +int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item) { for (Py_ssize_t i = 0; i < *len; i++) { int match = 0; if (items[i] == item) @@ -306,7 +306,7 @@ int vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item) { return 0; } -PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { +PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { if (index < 0) index += *len; @@ -323,12 +323,12 @@ PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index) { return item; } -PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *args) { +PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args) { Py_ssize_t index = -1; if (!PyArg_ParseTuple(args, "|n:pop", &index)) return NULL; - return vec_generic_pop(len, items, index); + return Vec_GenericPop(len, items, index); } // Module-level functions diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 9d35f97f4e922..1405574b123ef 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -369,13 +369,13 @@ static inline int check_float_error(PyObject *o) { return 0; } -PyObject *vec_type_to_str(size_t item_type, size_t depth); +PyObject *Vec_TypeToStr(size_t item_type, size_t depth); PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose); -PyObject *vec_generic_richcompare(Py_ssize_t *len, PyObject **items, +PyObject *Vec_GenericRichcompare(Py_ssize_t *len, PyObject **items, Py_ssize_t *other_len, PyObject **other_items, int op); -int vec_generic_remove(Py_ssize_t *len, PyObject **items, PyObject *item); -PyObject *vec_generic_pop_wrapper(Py_ssize_t *len, PyObject **items, PyObject *args); -PyObject *vec_generic_pop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); +int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item); +PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args); +PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); #endif From 228bcde8b05f4d85a303b0312f7d4fef484a0df8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:13:06 +0000 Subject: [PATCH 115/208] Refactor --- mypyc/lib-rt/vecs/vec_t_ext.c | 14 +++++++------- mypyc/lib-rt/vecs/vecs.c | 6 +++--- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 64a47349970a5..948c08a4e2af9 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -313,24 +313,24 @@ VecTExtPopResult VecVec_Pop(VecTExt v, Py_ssize_t index) { } static int -VecTExt_traverse(VecTObject *self, visitproc visit, void *arg) +VecVec_traverse(VecTObject *self, visitproc visit, void *arg) { Py_VISIT(self->vec.buf); return 0; } static int -VecTExt_clear(VecTObject *self) +VecVec_clear(VecTObject *self) { Py_CLEAR(self->vec.buf); return 0; } static void -VecTExt_dealloc(VecTObject *self) +VecVec_dealloc(VecTObject *self) { PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecTExt_dealloc) + Py_TRASHCAN_BEGIN(self, VecVec_dealloc) Py_CLEAR(self->vec.buf); Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END @@ -416,9 +416,9 @@ PyTypeObject VecTExtType = { .tp_basicsize = sizeof(VecTExtObject) - sizeof(PyObject *), .tp_itemsize = sizeof(PyObject *), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecTExt_traverse, - .tp_clear = (inquiry)VecTExt_clear, - .tp_dealloc = (destructor)VecTExt_dealloc, + .tp_traverse = (traverseproc)VecVec_traverse, + .tp_clear = (inquiry)VecVec_clear, + .tp_dealloc = (destructor)VecVec_dealloc, //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_repr, .tp_as_sequence = &VecTExtSequence, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index a996248d744f4..579b1f132bc0e 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -364,7 +364,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecT_Box(v, v.buf->item_type); - } else if (VecTExt_Check(vec)) { + } else if (VecVec_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) @@ -411,7 +411,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecT_Box(v, v.buf->item_type); - } else if (VecTExt_Check(vec)) { + } else if (VecVec_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecbufTExtItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) @@ -462,7 +462,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) return NULL; } result_item1 = r.f1; - } else if (VecTExt_Check(vec)) { + } else if (VecVec_Check(vec)) { VecTExt v = ((VecTExtObject *)vec)->vec; VecTExtPopResult r; r = VecVec_Pop(v, index); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 1405574b123ef..4658505c54d56 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -274,11 +274,11 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); // vec[t] operations (extended) -static inline int VecTExt_Check(PyObject *o) { +static inline int VecVec_Check(PyObject *o) { return o->ob_type == &VecTExtType; } -static inline int VecTExt_ItemCheck(VecTExt v, PyObject *it) { +static inline int VecVec_ItemCheck(VecTExt v, PyObject *it) { // TODO: vec[i64] item type if (it == Py_None && (v.buf->item_type & 1)) { return 1; From d83ca0509f393da8a5ebbd974c58963202cae0f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:16:06 +0000 Subject: [PATCH 116/208] Refactor --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_t_ext.c | 8 ++++---- mypyc/lib-rt/vecs/vecs.c | 8 ++++---- mypyc/lib-rt/vecs/vecs.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index ad42d5a4bbf55..8671239731cf9 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -184,7 +184,7 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { } int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - if (check_float_error(o)) + if (Vec_CheckFloatError(o)) return -1; VecI64 v = ((VecI64Object *)self)->vec; if ((size_t)i < (size_t)v.len) { diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 948c08a4e2af9..1c831b4e5a0e8 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -22,7 +22,7 @@ static VecTExt vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { return VecVec_Error(); buf->item_type = item_type; buf->depth = depth; - if (!vec_is_magic_item_type(item_type)) + if (!Vec_IsMagicItemType(item_type)) Py_INCREF(BUF_ITEM_TYPE(buf)); VecTExt res = { .buf = buf }; PyObject_GC_Track(buf); @@ -339,7 +339,7 @@ VecVec_dealloc(VecTObject *self) static int VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) { - if (!vec_is_magic_item_type(self->item_type)) + if (!Vec_IsMagicItemType(self->item_type)) Py_VISIT(BUF_ITEM_TYPE(self)); for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { Py_VISIT(self->items[i].buf); @@ -350,7 +350,7 @@ VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) static int VecbufTExt_clear(VecbufTExtObject *self) { - if (self->item_type && !vec_is_magic_item_type(self->item_type)) { + if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; } @@ -365,7 +365,7 @@ VecbufTExt_dealloc(VecbufTExtObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecbufTExt_dealloc) - if (self->item_type && !vec_is_magic_item_type(self->item_type)) { + if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; } diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 579b1f132bc0e..826850e9cbbc8 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -50,7 +50,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) static int VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) { - if (!vec_is_magic_item_type(self->item_type)) + if (!Vec_IsMagicItemType(self->item_type)) Py_VISIT((PyObject *)(self->item_type & ~1)); return 0; } @@ -58,7 +58,7 @@ VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) static void VecProxy_dealloc(VecProxy *self) { - if (self->item_type && !vec_is_magic_item_type(self->item_type)) { + if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF((PyObject *)(self->item_type & ~1)); self->item_type = 0; } @@ -342,7 +342,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return NULL; if (VecI64_Check(vec)) { - if (check_float_error(item)) + if (Vec_CheckFloatError(item)) return NULL; int64_t x = PyLong_AsLong(item); if (x == -1 && PyErr_Occurred()) { @@ -389,7 +389,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return NULL; if (VecI64_Check(vec)) { - if (check_float_error(item)) + if (Vec_CheckFloatError(item)) return NULL; int64_t x = PyLong_AsLong(item); if (x == -1 && PyErr_Occurred()) { diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 4658505c54d56..c2d3dabb7f74f 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -11,7 +11,7 @@ // Item type constants; must be even but not multiples of 4 (2 + 4 * n) #define VEC_ITEM_TYPE_I64 2 -inline size_t vec_is_magic_item_type(size_t item_type) { +inline size_t Vec_IsMagicItemType(size_t item_type) { return item_type & 2; } @@ -361,7 +361,7 @@ static inline PyObject *VecVec_BoxItem(VecTExt v, VecbufTExtItem item) { // Misc helpers -static inline int check_float_error(PyObject *o) { +static inline int Vec_CheckFloatError(PyObject *o) { if (PyFloat_Check(o)) { PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); return 1; From 0eaf3d3551365d96c88c55753296c2cc3cd4193f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:17:55 +0000 Subject: [PATCH 117/208] Refactor --- mypyc/lib-rt/vecs/vec_t_ext.c | 76 +++++++++++++++++----------------- mypyc/lib-rt/vecs/vecs.c | 14 +++---- mypyc/lib-rt/vecs/vecs.h | 78 +++++++++++++++++------------------ 3 files changed, 84 insertions(+), 84 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 1c831b4e5a0e8..630775c3983c2 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -9,14 +9,14 @@ #include #include "vecs.h" -static inline PyObject *box_vec_item_by_index(VecTExt v, Py_ssize_t index) { +static inline PyObject *box_vec_item_by_index(VecNested v, Py_ssize_t index) { return VecVec_BoxItem(v, v.buf->items[index]); } // Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the // return value. -static VecTExt vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { +static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); if (buf == NULL) return VecVec_Error(); @@ -24,13 +24,13 @@ static VecTExt vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { buf->depth = depth; if (!Vec_IsMagicItemType(item_type)) Py_INCREF(BUF_ITEM_TYPE(buf)); - VecTExt res = { .buf = buf }; + VecNested res = { .buf = buf }; PyObject_GC_Track(buf); return res; } -PyObject *VecVec_Box(VecTExt vec) { - VecTExtObject *obj = PyObject_GC_New(VecTExtObject, &VecTExtType); +PyObject *VecVec_Box(VecNested vec) { + VecNestedObject *obj = PyObject_GC_New(VecNestedObject, &VecNestedType); if (obj == NULL) return NULL; obj->vec = vec; @@ -38,9 +38,9 @@ PyObject *VecVec_Box(VecTExt vec) { return (PyObject *)obj; } -VecTExt VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { - if (obj->ob_type == &VecTExtType) { - VecTExt result = ((VecTExtObject *)obj)->vec; +VecNested VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { + if (obj->ob_type == &VecNestedType) { + VecNested result = ((VecNestedObject *)obj)->vec; if (result.buf->item_type == item_type && result.buf->depth == depth) { VEC_INCREF(result); // TODO: Should we borrow instead? return result; @@ -51,14 +51,14 @@ VecTExt VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { return VecVec_Error(); } -VecTExt VecVec_ConvertFromNested(VecbufTExtItem item) { - return (VecTExt) { item.len, (VecbufTExtObject *)item.buf }; +VecNested VecVec_ConvertFromNested(VecbufTExtItem item) { + return (VecNested) { item.len, (VecbufTExtObject *)item.buf }; } -VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { +VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; - VecTExt vec = vec_alloc(cap, item_type, depth); + VecNested vec = vec_alloc(cap, item_type, depth); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < cap; i++) { @@ -70,12 +70,12 @@ VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t dep } static PyObject *vec_repr(PyObject *self) { - VecTExt v = ((VecTExtObject *)self)->vec; + VecNested v = ((VecNestedObject *)self)->vec; return Vec_GenericRepr(self, v.buf->item_type, v.buf->depth, 1); } static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecTExt v = ((VecTExtObject *)o)->vec; + VecNested v = ((VecNestedObject *)o)->vec; if ((size_t)i < (size_t)v.len) { return box_vec_item_by_index(v, i); } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { @@ -86,7 +86,7 @@ static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { +VecNested VecVec_Slice(VecNested vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -100,7 +100,7 @@ VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecTExt res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); + VecNested res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -113,7 +113,7 @@ VecTExt VecVec_Slice(VecTExt vec, int64_t start, int64_t end) { } static PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecTExt vec = ((VecTExtObject *)self)->vec; + VecNested vec = ((VecNestedObject *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -131,7 +131,7 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecTExt res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); + VecNested res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -151,7 +151,7 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { } static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - VecTExt v = ((VecTExtObject *)self)->vec; + VecNested v = ((VecNestedObject *)self)->vec; if ((size_t)i + (size_t)v.len < (size_t)v.len) { i += v.len; } @@ -169,7 +169,7 @@ static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } } -static PyObject *compare_vec_eq(VecTExt x, VecTExt y, int op) { +static PyObject *compare_vec_eq(VecNested x, VecNested y, int op) { int cmp = 1; PyObject *res; if (x.len != y.len @@ -201,11 +201,11 @@ static PyObject *compare_vec_eq(VecTExt x, VecTExt y, int op) { PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { PyObject *res; if (op == Py_EQ || op == Py_NE) { - if (other->ob_type != &VecTExtType) { + if (other->ob_type != &VecNestedType) { res = op == Py_EQ ? Py_False : Py_True; } else { - return compare_vec_eq(((VecTExtObject *)self)->vec, - ((VecTExtObject *)other)->vec, op); + return compare_vec_eq(((VecNestedObject *)self)->vec, + ((VecNestedObject *)other)->vec, op); } } else res = Py_NotImplemented; @@ -214,7 +214,7 @@ PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { } // Steals reference to vec (but not x) -VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x) { +VecNested VecVec_Append(VecNested vec, VecbufTExtItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); if (vec.len < cap) { @@ -224,7 +224,7 @@ VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x) { } else { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here - VecTExt new = vec_alloc(new_size, vec.buf->item_type, vec.buf->depth); + VecNested new = vec_alloc(new_size, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(new)) { VEC_DECREF(x); return new; @@ -242,7 +242,7 @@ VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x) { } } -VecTExt VecVec_Remove(VecTExt self, VecbufTExtItem arg) { +VecNested VecVec_Remove(VecNested self, VecbufTExtItem arg) { VecbufTExtItem *items = self.buf->items; PyObject *boxed_arg = VecVec_BoxItem(self, arg); @@ -286,8 +286,8 @@ VecTExt VecVec_Remove(VecTExt self, VecbufTExtItem arg) { return VecVec_Error(); } -VecTExtPopResult VecVec_Pop(VecTExt v, Py_ssize_t index) { - VecTExtPopResult result; +VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { + VecNestedPopResult result; if (index < 0) index += v.len; @@ -378,15 +378,15 @@ VecbufTExt_dealloc(VecbufTExtObject *self) static Py_ssize_t vec_ext_length(PyObject *o) { // TODO: Type check o - return ((VecTExtObject *)o)->vec.len; + return ((VecNestedObject *)o)->vec.len; } -static PyMappingMethods VecTExtMapping = { +static PyMappingMethods VecNestedMapping = { .mp_length = vec_ext_length, .mp_subscript = vec_subscript, }; -static PySequenceMethods VecTExtSequence = { +static PySequenceMethods VecNestedSequence = { .sq_item = vec_get_item, .sq_ass_item = vec_ass_item, }; @@ -409,11 +409,11 @@ PyTypeObject VecbufTExtType = { .tp_dealloc = (destructor)VecbufTExt_dealloc, }; -PyTypeObject VecTExtType = { +PyTypeObject VecNestedType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecTExtObject) - sizeof(PyObject *), + .tp_basicsize = sizeof(VecNestedObject) - sizeof(PyObject *), .tp_itemsize = sizeof(PyObject *), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .tp_traverse = (traverseproc)VecVec_traverse, @@ -421,15 +421,15 @@ PyTypeObject VecTExtType = { .tp_dealloc = (destructor)VecVec_dealloc, //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecTExtSequence, - .tp_as_mapping = &VecTExtMapping, + .tp_as_sequence = &VecNestedSequence, + .tp_as_mapping = &VecNestedMapping, .tp_richcompare = vec_richcompare, .tp_methods = vec_methods, // TODO: free }; PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { - VecTExt v = vec_alloc(0, item_type, depth); + VecNested v = vec_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; @@ -464,8 +464,8 @@ PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable return VecVec_Box(v); } -VecTExtFeatures TExtFeatures = { - &VecTExtType, +VecNestedFeatures TExtFeatures = { + &VecNestedType, &VecbufTExtType, VecVec_New, VecVec_Box, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 826850e9cbbc8..96d94205fd5ce 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -15,7 +15,7 @@ typedef struct { // Tagged pointer to PyTypeObject *, lowest bit set for optional item type // Can also be one of magic VEC_ITEM_TYPE_* constants size_t item_type; - size_t depth; // Number of nested VecTExt or VecT types + size_t depth; // Number of nested VecNested or VecT types } VecProxy; static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) @@ -37,7 +37,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } } else { if (init == NULL) { - VecTExt vec = VecVec_New(0, 0, p->item_type, p->depth); + VecNested vec = VecVec_New(0, 0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; return VecVec_Box(vec); @@ -365,7 +365,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return NULL; return VecT_Box(v, v.buf->item_type); } else if (VecVec_Check(vec)) { - VecTExt v = ((VecTExtObject *)vec)->vec; + VecNested v = ((VecNestedObject *)vec)->vec; VecbufTExtItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; @@ -412,7 +412,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return NULL; return VecT_Box(v, v.buf->item_type); } else if (VecVec_Check(vec)) { - VecTExt v = ((VecTExtObject *)vec)->vec; + VecNested v = ((VecNestedObject *)vec)->vec; VecbufTExtItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; @@ -463,8 +463,8 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) } result_item1 = r.f1; } else if (VecVec_Check(vec)) { - VecTExt v = ((VecTExtObject *)vec)->vec; - VecTExtPopResult r; + VecNested v = ((VecNestedObject *)vec)->vec; + VecNestedPopResult r; r = VecVec_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; @@ -549,7 +549,7 @@ PyInit_vecs(void) if (PyType_Ready(&VecbufTType) < 0) return NULL; - if (PyType_Ready(&VecTExtType) < 0) + if (PyType_Ready(&VecNestedType) < 0) return NULL; if (PyType_Ready(&VecbufTExtType) < 0) return NULL; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index c2d3dabb7f74f..0d0788157efcf 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -63,10 +63,10 @@ typedef struct _VecT { VecbufTObject *buf; } VecT; -typedef struct _VecTExt { +typedef struct _VecNested { Py_ssize_t len; VecbufTExtObject *buf; -} VecTExt; +} VecNested; // Boxed vec objects @@ -91,10 +91,10 @@ typedef struct _VecTObject { } VecTObject; // Extended generic vec type: vec[t | None], vec[vec[...]], etc. -typedef struct _VecTExtObject { +typedef struct _VecNestedObject { PyObject_HEAD - VecTExt vec; -} VecTExtObject; + VecNested vec; +} VecNestedObject; #ifndef MYPYC_DECLARED_tuple_T2V88 @@ -163,39 +163,39 @@ typedef struct _VecTFeatures { #ifndef MYPYC_DECLARED_tuple_T2VvVi #define MYPYC_DECLARED_tuple_T2VvVi typedef struct tuple_T2VvVi { - VecTExt f0; + VecNested f0; VecbufTExtItem f1; } tuple_T2VvVi; static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; #endif -typedef tuple_T2VvVi VecTExtPopResult; +typedef tuple_T2VvVi VecNestedPopResult; // vec[T] operations for complex item types + type object (stored in a capsule) // // T can be T | None or vec[T] (or a combination of these) -typedef struct _VecTExtFeatures { +typedef struct _VecNestedFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; - VecTExt (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); - PyObject *(*box)(VecTExt); - VecTExt (*unbox)(PyObject *, size_t, size_t depth); - VecTExt (*convert_from_nested)(VecbufTExtItem); - VecTExt (*append)(VecTExt, VecbufTExtItem); - VecTExtPopResult (*pop)(VecTExt, Py_ssize_t); - VecTExt (*remove)(VecTExt, VecbufTExtItem); + VecNested (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); + PyObject *(*box)(VecNested); + VecNested (*unbox)(PyObject *, size_t, size_t depth); + VecNested (*convert_from_nested)(VecbufTExtItem); + VecNested (*append)(VecNested, VecbufTExtItem); + VecNestedPopResult (*pop)(VecNested, Py_ssize_t); + VecNested (*remove)(VecNested, VecbufTExtItem); // TODO: Py_ssize_t - VecTExt (*slice)(VecTExt, int64_t, int64_t); + VecNested (*slice)(VecNested, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); // iter? -} VecTExtFeatures; +} VecNestedFeatures; typedef struct { VecI64Features *i64; VecTFeatures *t; - VecTExtFeatures *t_ext; + VecNestedFeatures *t_ext; } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) @@ -216,8 +216,8 @@ inline VecT VecT_Error() { return v; } -inline VecTExt VecVec_Error() { - VecTExt v = { .len = -1 }; +inline VecNested VecVec_Error() { + VecNested v = { .len = -1 }; return v; } @@ -229,12 +229,12 @@ extern PyTypeObject VecbufTExtType; extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; -extern PyTypeObject VecTExtType; +extern PyTypeObject VecNestedType; extern VecI64Features I64Features; extern PyTypeObject *I64TypeObj; extern VecTFeatures TFeatures; -extern VecTExtFeatures TExtFeatures; +extern VecNestedFeatures TExtFeatures; // vec[i64] operations @@ -275,19 +275,19 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); // vec[t] operations (extended) static inline int VecVec_Check(PyObject *o) { - return o->ob_type == &VecTExtType; + return o->ob_type == &VecNestedType; } -static inline int VecVec_ItemCheck(VecTExt v, PyObject *it) { +static inline int VecVec_ItemCheck(VecNested v, PyObject *it) { // TODO: vec[i64] item type if (it == Py_None && (v.buf->item_type & 1)) { return 1; } else if (v.buf->depth == 1 && it->ob_type == &VecTType - && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type) { + && ((VecNestedObject *)it)->vec.buf->item_type == v.buf->item_type) { return 1; - } else if (it->ob_type == &VecTExtType - && ((VecTExtObject *)it)->vec.buf->depth == v.buf->depth - 1 - && ((VecTExtObject *)it)->vec.buf->item_type == v.buf->item_type) { + } else if (it->ob_type == &VecNestedType + && ((VecNestedObject *)it)->vec.buf->depth == v.buf->depth - 1 + && ((VecNestedObject *)it)->vec.buf->item_type == v.buf->item_type) { return 1; } else { // TODO: better error message @@ -296,21 +296,21 @@ static inline int VecVec_ItemCheck(VecTExt v, PyObject *it) { } } -VecTExt VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); +VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable); -PyObject *VecVec_Box(VecTExt); -VecTExt VecVec_Append(VecTExt vec, VecbufTExtItem x); -VecTExt VecVec_Remove(VecTExt vec, VecbufTExtItem x); -VecTExtPopResult VecVec_Pop(VecTExt v, Py_ssize_t index); +PyObject *VecVec_Box(VecNested); +VecNested VecVec_Append(VecNested vec, VecbufTExtItem x); +VecNested VecVec_Remove(VecNested vec, VecbufTExtItem x); +VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. -static inline int VecVec_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *unboxed) { +static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecbufTExtItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { // TODO: vec[i64] if (item->ob_type == &VecTType) { - VecTExtObject *o = (VecTExtObject *)item; + VecNestedObject *o = (VecNestedObject *)item; if (o->vec.buf->item_type == v.buf->item_type) { unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; @@ -322,8 +322,8 @@ static inline int VecVec_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *un unboxed->buf = (PyObject *)o->vec.buf; return 0; } - } else if (item->ob_type == &VecTExtType) { - VecTExtObject *o = (VecTExtObject *)item; + } else if (item->ob_type == &VecNestedType) { + VecNestedObject *o = (VecNestedObject *)item; if (o->vec.buf->depth == v.buf->depth - 1 && o->vec.buf->item_type == v.buf->item_type) { unboxed->len = o->vec.len; @@ -336,13 +336,13 @@ static inline int VecVec_UnboxItem(VecTExt v, PyObject *item, VecbufTExtItem *un return -1; } -static inline PyObject *VecVec_BoxItem(VecTExt v, VecbufTExtItem item) { +static inline PyObject *VecVec_BoxItem(VecNested v, VecbufTExtItem item) { if (item.len < 0) Py_RETURN_NONE; Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec - VecTExt v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; + VecNested v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; return VecVec_Box(v); } else { // Item is a non-nested vec From e2f521b63bb1a64632e85f6dd60131a20c54c5b5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:43:38 +0000 Subject: [PATCH 118/208] Refactor --- mypyc/lib-rt/vecs/vec_i64.c | 12 ++++----- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_t_ext.c | 50 +++++++++++++++++------------------ mypyc/lib-rt/vecs/vecs.c | 8 +++--- mypyc/lib-rt/vecs/vecs.h | 46 ++++++++++++++++---------------- 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 8671239731cf9..33fa27b4aae80 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -5,12 +5,12 @@ // Alloc a partially initialized vec. Caller *must* initialize len. static VecI64 vec_alloc(Py_ssize_t size) { - VecbufI64Object *buf; + VecI64BufObject *buf; /* TODO: Check for overflow */ if (size == 0) { buf = NULL; } else { - buf = PyObject_NewVar(VecbufI64Object, &VecbufI64Type, size); + buf = PyObject_NewVar(VecI64BufObject, &VecI64BufType, size); if (buf == NULL) return VecI64_Error(); } @@ -43,8 +43,8 @@ VecI64 VecI64_Unbox(PyObject *obj) { } } -VecI64 VecI64_ConvertFromNested(VecbufTExtItem item) { - return (VecI64) { item.len, (VecbufI64Object *)item.buf }; +VecI64 VecI64_ConvertFromNested(VecNestedBufItem item) { + return (VecI64) { item.len, (VecI64BufObject *)item.buf }; } VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { @@ -312,7 +312,7 @@ static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecbufI64Type = { +PyTypeObject VecI64BufType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf[i64]", .tp_doc = "vec doc", @@ -342,7 +342,7 @@ PyTypeObject VecI64Type = { VecI64Features I64Features = { &VecI64Type, - &VecbufI64Type, + &VecI64BufType, VecI64_New, VecI64_Box, VecI64_Unbox, diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index b71ef52cdaf57..5f09f593e6a61 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -64,7 +64,7 @@ VecT VecT_Unbox(PyObject *obj, size_t item_type) { return VecT_Error(); } -VecT VecT_ConvertFromNested(VecbufTExtItem item) { +VecT VecT_ConvertFromNested(VecNestedBufItem item) { return (VecT) { item.len, (VecbufTObject *)item.buf }; } diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_t_ext.c index 630775c3983c2..aca9051a5b2cb 100644 --- a/mypyc/lib-rt/vecs/vec_t_ext.c +++ b/mypyc/lib-rt/vecs/vec_t_ext.c @@ -17,7 +17,7 @@ static inline PyObject *box_vec_item_by_index(VecNested v, Py_ssize_t index) { // Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the // return value. static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { - VecbufTExtObject *buf = PyObject_GC_NewVar(VecbufTExtObject, &VecbufTExtType, size); + VecNestedBufObject *buf = PyObject_GC_NewVar(VecNestedBufObject, &VecNestedBufType, size); if (buf == NULL) return VecVec_Error(); buf->item_type = item_type; @@ -51,8 +51,8 @@ VecNested VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { return VecVec_Error(); } -VecNested VecVec_ConvertFromNested(VecbufTExtItem item) { - return (VecNested) { item.len, (VecbufTExtObject *)item.buf }; +VecNested VecVec_ConvertFromNested(VecNestedBufItem item) { + return (VecNested) { item.len, (VecNestedBufObject *)item.buf }; } VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { @@ -105,7 +105,7 @@ VecNested VecVec_Slice(VecNested vec, int64_t start, int64_t end) { return res; res.len = slicelength; for (Py_ssize_t i = 0; i < slicelength; i++) { - VecbufTExtItem item = vec.buf->items[start + i]; + VecNestedBufItem item = vec.buf->items[start + i]; Py_XINCREF(item.buf); res.buf->items[i] = item; } @@ -137,7 +137,7 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { res.len = slicelength; Py_ssize_t j = start; for (Py_ssize_t i = 0; i < slicelength; i++) { - VecbufTExtItem item = vec.buf->items[j]; + VecNestedBufItem item = vec.buf->items[j]; Py_INCREF(item.buf); res.buf->items[i] = item; j += step; @@ -156,7 +156,7 @@ static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { i += v.len; } if ((size_t)i < (size_t)v.len) { - VecbufTExtItem item; + VecNestedBufItem item; if (VecVec_UnboxItem(v, o, &item) < 0) return -1; VEC_INCREF(item); @@ -214,7 +214,7 @@ PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { } // Steals reference to vec (but not x) -VecNested VecVec_Append(VecNested vec, VecbufTExtItem x) { +VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); if (vec.len < cap) { @@ -230,11 +230,11 @@ VecNested VecVec_Append(VecNested vec, VecbufTExtItem x) { return new; } // Copy items to new vec. - memcpy(new.buf->items, vec.buf->items, sizeof(VecbufTExtItem) * vec.len); + memcpy(new.buf->items, vec.buf->items, sizeof(VecNestedBufItem) * vec.len); // TODO: How to safely represent deleted items? - memset(new.buf->items + vec.len, 0, sizeof(VecbufTExtItem) * (new_size - vec.len)); + memset(new.buf->items + vec.len, 0, sizeof(VecNestedBufItem) * (new_size - vec.len)); // Clear the items in the old vec. We avoid reference count manipulation. - memset(vec.buf->items, 0, sizeof(VecbufTExtItem) * vec.len); + memset(vec.buf->items, 0, sizeof(VecNestedBufItem) * vec.len); new.buf->items[vec.len] = x; new.len = vec.len + 1; VEC_DECREF(vec); @@ -242,8 +242,8 @@ VecNested VecVec_Append(VecNested vec, VecbufTExtItem x) { } } -VecNested VecVec_Remove(VecNested self, VecbufTExtItem arg) { - VecbufTExtItem *items = self.buf->items; +VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { + VecNestedBufItem *items = self.buf->items; PyObject *boxed_arg = VecVec_BoxItem(self, arg); if (boxed_arg == NULL) @@ -300,7 +300,7 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { return result; } - VecbufTExtItem *items = v.buf->items; + VecNestedBufItem *items = v.buf->items; result.f1 = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; @@ -337,7 +337,7 @@ VecVec_dealloc(VecTObject *self) } static int -VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) +VecNestedBuf_traverse(VecNestedBufObject *self, visitproc visit, void *arg) { if (!Vec_IsMagicItemType(self->item_type)) Py_VISIT(BUF_ITEM_TYPE(self)); @@ -348,7 +348,7 @@ VecbufTExt_traverse(VecbufTExtObject *self, visitproc visit, void *arg) } static int -VecbufTExt_clear(VecbufTExtObject *self) +VecNestedBuf_clear(VecNestedBufObject *self) { if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); @@ -361,10 +361,10 @@ VecbufTExt_clear(VecbufTExtObject *self) } static void -VecbufTExt_dealloc(VecbufTExtObject *self) +VecNestedBuf_dealloc(VecNestedBufObject *self) { PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecbufTExt_dealloc) + Py_TRASHCAN_BEGIN(self, VecNestedBuf_dealloc) if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; @@ -395,18 +395,18 @@ static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecbufTExtType = { +PyTypeObject VecNestedBufType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecbufTExtObject) - sizeof(VecbufTExtItem), - .tp_itemsize = sizeof(VecbufTExtItem), + .tp_basicsize = sizeof(VecNestedBufObject) - sizeof(VecNestedBufItem), + .tp_itemsize = sizeof(VecNestedBufItem), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecbufTExt_traverse, + .tp_traverse = (traverseproc)VecNestedBuf_traverse, //.tp_new = vecbuf_i64_new, //?? .tp_free = PyObject_GC_Del, - .tp_clear = (inquiry)VecbufTExt_clear, - .tp_dealloc = (destructor)VecbufTExt_dealloc, + .tp_clear = (inquiry)VecNestedBuf_clear, + .tp_dealloc = (destructor)VecNestedBuf_dealloc, }; PyTypeObject VecNestedType = { @@ -441,7 +441,7 @@ PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { - VecbufTExtItem vecitem; + VecNestedBufItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) { Py_DECREF(iter); VEC_DECREF(v); @@ -466,7 +466,7 @@ PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable VecNestedFeatures TExtFeatures = { &VecNestedType, - &VecbufTExtType, + &VecNestedBufType, VecVec_New, VecVec_Box, VecVec_Unbox, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 96d94205fd5ce..e9261fbc0f033 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -366,7 +366,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return VecT_Box(v, v.buf->item_type); } else if (VecVec_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; - VecbufTExtItem vecitem; + VecNestedBufItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); @@ -413,7 +413,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return VecT_Box(v, v.buf->item_type); } else if (VecVec_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; - VecbufTExtItem vecitem; + VecNestedBufItem vecitem; if (VecVec_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); @@ -551,12 +551,12 @@ PyInit_vecs(void) if (PyType_Ready(&VecNestedType) < 0) return NULL; - if (PyType_Ready(&VecbufTExtType) < 0) + if (PyType_Ready(&VecNestedBufType) < 0) return NULL; if (PyType_Ready(&VecI64Type) < 0) return NULL; - if (PyType_Ready(&VecbufI64Type) < 0) + if (PyType_Ready(&VecI64BufType) < 0) return NULL; PyObject *m = PyModule_Create(&vecsmodule); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 0d0788157efcf..0d56176eb18f5 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -20,10 +20,10 @@ inline size_t Vec_IsMagicItemType(size_t item_type) { // vecbuf[i64] -typedef struct _VecbufI64Object { +typedef struct _VecI64BufObject { PyObject_VAR_HEAD int64_t items[1]; -} VecbufI64Object; +} VecI64BufObject; // Simple generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecbufTObject { @@ -33,21 +33,21 @@ typedef struct _VecbufTObject { PyObject *items[1]; } VecbufTObject; -typedef struct _VecbufTExtItem { +typedef struct _VecNestedBufItem { Py_ssize_t len; PyObject *buf; -} VecbufTExtItem; +} VecNestedBufItem; // Nested vec type: vec[vec[...]], vec[vec[...] | None], etc. -typedef struct _VecbufTExtObject { +typedef struct _VecNestedBufObject { PyObject_VAR_HEAD // Tagged pointer to PyTypeObject *. Lowest bit is set for optional item type. // The second lowest bit is set for a packed item type (VEC_ITEM_TYPE_*). size_t item_type; // Number of nested vec types (of any kind, at least 1) size_t depth; - VecbufTExtItem items[1]; -} VecbufTExtObject; + VecNestedBufItem items[1]; +} VecNestedBufObject; // Unboxed vec objects @@ -55,7 +55,7 @@ typedef struct _VecbufTExtObject { typedef struct _VecI64 { Py_ssize_t len; - VecbufI64Object *buf; + VecI64BufObject *buf; } VecI64; typedef struct _VecT { @@ -65,7 +65,7 @@ typedef struct _VecT { typedef struct _VecNested { Py_ssize_t len; - VecbufTExtObject *buf; + VecNestedBufObject *buf; } VecNested; @@ -115,7 +115,7 @@ typedef struct _VecI64Features { VecI64 (*alloc)(Py_ssize_t, Py_ssize_t); PyObject *(*box)(VecI64); VecI64 (*unbox)(PyObject *); - VecI64 (*convert_from_nested)(VecbufTExtItem); + VecI64 (*convert_from_nested)(VecNestedBufItem); VecI64 (*append)(VecI64, int64_t); VecI64PopResult (*pop)(VecI64, Py_ssize_t); VecI64 (*remove)(VecI64, int64_t); @@ -147,7 +147,7 @@ typedef struct _VecTFeatures { VecT (*alloc)(Py_ssize_t, Py_ssize_t, size_t); PyObject *(*box)(VecT, size_t); VecT (*unbox)(PyObject *, size_t); - VecT (*convert_from_nested)(VecbufTExtItem); + VecT (*convert_from_nested)(VecNestedBufItem); VecT (*append)(VecT, PyObject *, size_t); VecTPopResult (*pop)(VecT, Py_ssize_t); VecT (*remove)(VecT, PyObject *); @@ -164,7 +164,7 @@ typedef struct _VecTFeatures { #define MYPYC_DECLARED_tuple_T2VvVi typedef struct tuple_T2VvVi { VecNested f0; - VecbufTExtItem f1; + VecNestedBufItem f1; } tuple_T2VvVi; static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; #endif @@ -180,10 +180,10 @@ typedef struct _VecNestedFeatures { VecNested (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); PyObject *(*box)(VecNested); VecNested (*unbox)(PyObject *, size_t, size_t depth); - VecNested (*convert_from_nested)(VecbufTExtItem); - VecNested (*append)(VecNested, VecbufTExtItem); + VecNested (*convert_from_nested)(VecNestedBufItem); + VecNested (*append)(VecNested, VecNestedBufItem); VecNestedPopResult (*pop)(VecNested, Py_ssize_t); - VecNested (*remove)(VecNested, VecbufTExtItem); + VecNested (*remove)(VecNested, VecNestedBufItem); // TODO: Py_ssize_t VecNested (*slice)(VecNested, int64_t, int64_t); // PyObject *(*extend)(PyObject *, PyObject *); @@ -223,9 +223,9 @@ inline VecNested VecVec_Error() { // Type objects -extern PyTypeObject VecbufI64Type; +extern PyTypeObject VecI64BufType; extern PyTypeObject VecbufTType; -extern PyTypeObject VecbufTExtType; +extern PyTypeObject VecNestedBufType; extern PyTypeObject VecI64Type; extern PyTypeObject VecTType; @@ -299,13 +299,13 @@ static inline int VecVec_ItemCheck(VecNested v, PyObject *it) { VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable); PyObject *VecVec_Box(VecNested); -VecNested VecVec_Append(VecNested vec, VecbufTExtItem x); -VecNested VecVec_Remove(VecNested vec, VecbufTExtItem x); +VecNested VecVec_Append(VecNested vec, VecNestedBufItem x); +VecNested VecVec_Remove(VecNested vec, VecNestedBufItem x); VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. -static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecbufTExtItem *unboxed) { +static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { // TODO: vec[i64] @@ -336,20 +336,20 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecbufTExtItem * return -1; } -static inline PyObject *VecVec_BoxItem(VecNested v, VecbufTExtItem item) { +static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { if (item.len < 0) Py_RETURN_NONE; Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec - VecNested v = { .len = item.len, .buf = (VecbufTExtObject *)item.buf }; + VecNested v = { .len = item.len, .buf = (VecNestedBufObject *)item.buf }; return VecVec_Box(v); } else { // Item is a non-nested vec size_t item_type = v.buf->item_type; if (item_type == VEC_ITEM_TYPE_I64) { // vec[i64] - VecI64 v = { .len = item.len, .buf = (VecbufI64Object *)item.buf }; + VecI64 v = { .len = item.len, .buf = (VecI64BufObject *)item.buf }; return VecI64_Box(v); } else { // Generic vec[t] From e91f13ade8247210bd00bdd85205f2f5e6714c5f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:44:43 +0000 Subject: [PATCH 119/208] Refactor --- mypyc/lib-rt/vecs/vec_t.c | 28 ++++++++++++++-------------- mypyc/lib-rt/vecs/vecs.c | 2 +- mypyc/lib-rt/vecs/vecs.h | 10 +++++----- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 5f09f593e6a61..e25596c53e972 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -10,8 +10,8 @@ #include #include "vecs.h" -static inline VecbufTObject *alloc_buf(Py_ssize_t size, size_t item_type) { - VecbufTObject *buf = PyObject_GC_NewVar(VecbufTObject, &VecbufTType, size); +static inline VecTBufObject *alloc_buf(Py_ssize_t size, size_t item_type) { + VecTBufObject *buf = PyObject_GC_NewVar(VecTBufObject, &VecTBufType, size); if (buf == NULL) return NULL; buf->item_type = item_type; @@ -23,7 +23,7 @@ static inline VecbufTObject *alloc_buf(Py_ssize_t size, size_t item_type) { // Alloc a partially initialized vec. Caller *must* immediately initialize len (and buf->items // if size > 0). static VecT vec_alloc(Py_ssize_t size, size_t item_type) { - VecbufTObject *buf; + VecTBufObject *buf; if (size == 0) { buf = NULL; @@ -65,7 +65,7 @@ VecT VecT_Unbox(PyObject *obj, size_t item_type) { } VecT VecT_ConvertFromNested(VecNestedBufItem item) { - return (VecT) { item.len, (VecbufTObject *)item.buf }; + return (VecT) { item.len, (VecTBufObject *)item.buf }; } VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { @@ -322,7 +322,7 @@ VecT_dealloc(VecTObject *self) } static int -VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) +VecTBuf_traverse(VecTBufObject *self, visitproc visit, void *arg) { Py_VISIT(BUF_ITEM_TYPE(self)); for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { @@ -332,7 +332,7 @@ VecbufT_traverse(VecbufTObject *self, visitproc visit, void *arg) } static int -VecbufT_clear(VecbufTObject *self) +VecTBuf_clear(VecTBufObject *self) { if (self->item_type) { Py_DECREF(BUF_ITEM_TYPE(self)); @@ -345,10 +345,10 @@ VecbufT_clear(VecbufTObject *self) } static void -VecbufT_dealloc(VecbufTObject *self) +VecTBuf_dealloc(VecTBufObject *self) { PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecbufT_dealloc) + Py_TRASHCAN_BEGIN(self, VecTBuf_dealloc) if (self->item_type) { Py_DECREF(BUF_ITEM_TYPE(self)); self->item_type = 0; @@ -379,18 +379,18 @@ static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecbufTType = { +PyTypeObject VecTBufType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecbufTObject) - sizeof(PyObject *), + .tp_basicsize = sizeof(VecTBufObject) - sizeof(PyObject *), .tp_itemsize = sizeof(PyObject *), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecbufT_traverse, + .tp_traverse = (traverseproc)VecTBuf_traverse, //.tp_new = vecbuf_i64_new, //?? .tp_free = PyObject_GC_Del, - .tp_clear = (inquiry)VecbufT_clear, - .tp_dealloc = (destructor)VecbufT_dealloc, + .tp_clear = (inquiry)VecTBuf_clear, + .tp_dealloc = (destructor)VecTBuf_dealloc, }; PyTypeObject VecTType = { @@ -449,7 +449,7 @@ PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable) { VecTFeatures TFeatures = { &VecTType, - &VecbufTType, + &VecTBufType, VecT_New, VecT_Box, VecT_Unbox, diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index e9261fbc0f033..74bce9bcc20a6 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -546,7 +546,7 @@ PyInit_vecs(void) if (PyType_Ready(&VecTType) < 0) return NULL; - if (PyType_Ready(&VecbufTType) < 0) + if (PyType_Ready(&VecTBufType) < 0) return NULL; if (PyType_Ready(&VecNestedType) < 0) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 0d56176eb18f5..459722072e3e5 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -26,12 +26,12 @@ typedef struct _VecI64BufObject { } VecI64BufObject; // Simple generic vecbuf: vecbuf[t] when t is a type object -typedef struct _VecbufTObject { +typedef struct _VecTBufObject { PyObject_VAR_HEAD // Tagged pointer to PyTypeObject *. The lowest bit is 1 for optional item type. size_t item_type; PyObject *items[1]; -} VecbufTObject; +} VecTBufObject; typedef struct _VecNestedBufItem { Py_ssize_t len; @@ -60,7 +60,7 @@ typedef struct _VecI64 { typedef struct _VecT { Py_ssize_t len; - VecbufTObject *buf; + VecTBufObject *buf; } VecT; typedef struct _VecNested { @@ -224,7 +224,7 @@ inline VecNested VecVec_Error() { // Type objects extern PyTypeObject VecI64BufType; -extern PyTypeObject VecbufTType; +extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; extern PyTypeObject VecI64Type; @@ -353,7 +353,7 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { return VecI64_Box(v); } else { // Generic vec[t] - VecT v = { .len = item.len, .buf = (VecbufTObject *)item.buf }; + VecT v = { .len = item.len, .buf = (VecTBufObject *)item.buf }; return VecT_Box(v, item_type); } } From d313339189fe84ab4889a292bae1d01a5fe038ef Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:46:06 +0000 Subject: [PATCH 120/208] Rename file --- mypyc/lib-rt/vecs/{vec_t_ext.c => vec_nested.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mypyc/lib-rt/vecs/{vec_t_ext.c => vec_nested.c} (100%) diff --git a/mypyc/lib-rt/vecs/vec_t_ext.c b/mypyc/lib-rt/vecs/vec_nested.c similarity index 100% rename from mypyc/lib-rt/vecs/vec_t_ext.c rename to mypyc/lib-rt/vecs/vec_nested.c From 3caac005573bb7de1db5f67dc5c3c0c5bd2078b9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:47:17 +0000 Subject: [PATCH 121/208] Remove unused function VecVec_ItemCheck --- mypyc/lib-rt/vecs/vecs.h | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 459722072e3e5..025adacaba47e 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -278,24 +278,6 @@ static inline int VecVec_Check(PyObject *o) { return o->ob_type == &VecNestedType; } -static inline int VecVec_ItemCheck(VecNested v, PyObject *it) { - // TODO: vec[i64] item type - if (it == Py_None && (v.buf->item_type & 1)) { - return 1; - } else if (v.buf->depth == 1 && it->ob_type == &VecTType - && ((VecNestedObject *)it)->vec.buf->item_type == v.buf->item_type) { - return 1; - } else if (it->ob_type == &VecNestedType - && ((VecNestedObject *)it)->vec.buf->depth == v.buf->depth - 1 - && ((VecNestedObject *)it)->vec.buf->item_type == v.buf->item_type) { - return 1; - } else { - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); - return 0; - } -} - VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable); PyObject *VecVec_Box(VecNested); From d97f7e1e337caaf1faacf7994eae50feb078072b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:53:07 +0000 Subject: [PATCH 122/208] Update comments --- mypyc/lib-rt/vecs/vec_nested.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 7 ++++--- mypyc/lib-rt/vecs/vecs.h | 12 +++++------- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index aca9051a5b2cb..da47491b1f4ca 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -2,7 +2,7 @@ // // Examples of types supported: // - vec[vec[i64]] -// - vec[vec[str]] +// - vec[vec[vec[str]]] // - vec[vec[str | None]] #define PY_SSIZE_T_CLEAN diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index e25596c53e972..b025f3974686e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -1,8 +1,9 @@ -// Implementation of generic vec[t], when t is a plain type object. +// Implementation of generic vec[t], when t is a plain type object (possibly optional). // // Examples of types supported: // // - vec[str] +// - vec[str | None] // - vec[object] // - vec[UserClass] @@ -20,8 +21,8 @@ static inline VecTBufObject *alloc_buf(Py_ssize_t size, size_t item_type) { return buf; } -// Alloc a partially initialized vec. Caller *must* immediately initialize len (and buf->items -// if size > 0). +// Alloc a partially initialized vec. Caller *must* immediately initialize len, and buf->items +// if size > 0. static VecT vec_alloc(Py_ssize_t size, size_t item_type) { VecTBufObject *buf; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 025adacaba47e..267f2c864080a 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -138,9 +138,9 @@ static tuple_T2VOO tuple_undefined_T2VOO = { { -1, NULL } , NULL }; typedef tuple_T2VOO VecTPopResult; -// vec[T] operations + type object (stored in a capsule) +// vec[T] operations + type objects (stored in a capsule) // -// T is a class type +// T is a class type or class type | None typedef struct _VecTFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; @@ -171,9 +171,7 @@ static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; typedef tuple_T2VvVi VecNestedPopResult; -// vec[T] operations for complex item types + type object (stored in a capsule) -// -// T can be T | None or vec[T] (or a combination of these) +// Nested vec operations + type objects (stored in a capsule) typedef struct _VecNestedFeatures { PyTypeObject *boxed_type; PyTypeObject *buf_type; @@ -247,7 +245,7 @@ VecI64 VecI64_Append(VecI64, int64_t x); VecI64 VecI64_Remove(VecI64, int64_t x); VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); -// vec[t] operations (simple) +// vec[t] operations static inline int VecT_Check(PyObject *o) { return o->ob_type == &VecTType; @@ -272,7 +270,7 @@ VecT VecT_Append(VecT vec, PyObject *x, size_t item_type); VecT VecT_Remove(VecT vec, PyObject *x); VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); -// vec[t] operations (extended) +// Nested vec operations static inline int VecVec_Check(PyObject *o) { return o->ob_type == &VecNestedType; From 40d79cef7ded694db123c6173f8dedebe91226e9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 27 Dec 2023 09:55:53 +0000 Subject: [PATCH 123/208] Refactor --- mypyc/lib-rt/vecs/vecs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 267f2c864080a..c40cb88a81cd4 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -193,7 +193,7 @@ typedef struct _VecNestedFeatures { typedef struct { VecI64Features *i64; VecTFeatures *t; - VecNestedFeatures *t_ext; + VecNestedFeatures *nested; } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) From 7050fa523deb4e1049cbc858800c5cd76bb9f569 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:20:05 +0000 Subject: [PATCH 124/208] WIP start working on vec item type specialization template --- mypyc/lib-rt/vecs/vec_template.c | 367 +++++++++++++++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 mypyc/lib-rt/vecs/vec_template.c diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c new file mode 100644 index 0000000000000..e0097bb848c14 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -0,0 +1,367 @@ +// NOTE: This file can't be compiled on its own, it must be #included +// with certain #defines set, as described below. +// +// Implementation of a vec class specialized to a specific item type, such +// as vec[i64] or vec[float]. Assume that certain #defines are provided that +// provide all the item type specific definitions: +// +// PREFIX type name prefix used in definitions (e.g. VecI64) +// VECTYPE C struct used for the vec (e.g. VecI64) +// BOXTYPE PyTypeObject used for a boxed vec (e.g. VecI64Object) +// BUFOBJ C struct used for the buffer object (e.g. VecI64BufObject) +// BUFTYPE PyTypeObject used for the buffer object (e.g. VecI64BufType) + +#define PY_SSIZE_T_CLEAN +#include +#include "vecs.h" + +// Alloc a partially initialized vec. Caller *must* initialize len. +static VecI64 vec_alloc(Py_ssize_t size) +{ + VecI64BufObject *buf; + /* TODO: Check for overflow */ + if (size == 0) { + buf = NULL; + } else { + buf = PyObject_NewVar(VecI64BufObject, &VecI64BufType, size); + if (buf == NULL) + return VecI64_Error(); + } + VecI64 res = { .buf = buf }; + return res; +} + +static void vec_dealloc(VecI64Object *self) { + Py_CLEAR(self->vec.buf); + PyObject_Del(self); +} + +PyObject *VecI64_Box(VecI64 vec) { + VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); + if (obj == NULL) + return NULL; + obj->vec = vec; + return (PyObject *)obj; +} + +VecI64 VecI64_Unbox(PyObject *obj) { + if (obj->ob_type == &VecI64Type) { + VecI64 result = ((VecI64Object *)obj)->vec; + VEC_INCREF(result); // TODO: Should we borrow instead? + return result; + } else { + // TODO: Better error message + PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); + return VecI64_Error(); + } +} + +VecI64 VecI64_ConvertFromNested(VecNestedBufItem item) { + return (VecI64) { item.len, (VecI64BufObject *)item.buf }; +} + +VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { + if (cap < size) + size = cap; + VecI64 vec = vec_alloc(cap); + if (VEC_IS_ERROR(vec)) + return vec; + for (Py_ssize_t i = 0; i < cap; i++) { + vec.buf->items[i] = 0; + } + vec.len = size; + return vec; +} + +PyObject *VecI64_FromIterable(PyObject *iterable) { + VecI64 v = vec_alloc(0); + if (VEC_IS_ERROR(v)) + return NULL; + v.len = 0; + + PyObject *iter = PyObject_GetIter(iterable); + if (iter == NULL) { + VEC_DECREF(v); + return NULL; + } + PyObject *item; + while ((item = PyIter_Next(iter)) != NULL) { + int64_t x = PyLong_AsLongLong(item); + Py_DECREF(item); + if (x == -1 && PyErr_Occurred()) { + Py_DECREF(iter); + VEC_DECREF(v); + return NULL; + } + v = VecI64_Append(v, x); + if (VEC_IS_ERROR(v)) { + Py_DECREF(iter); + VEC_DECREF(v); + return NULL; + } + } + Py_DECREF(iter); + if (PyErr_Occurred()) { + VEC_DECREF(v); + return NULL; + } + return VecI64_Box(v); +} + +PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { + static char *kwlist[] = {"", NULL}; + PyObject *init = NULL; + if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { + return NULL; + } + if (init == NULL) { + return VecI64_Box(VecI64_New(0, 0)); + } else { + return (PyObject *)VecI64_FromIterable(init); + } +} + +PyObject *vec_repr(PyObject *self) { + return Vec_GenericRepr(self, (size_t)I64TypeObj, 0, 1); +} + +PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { + VecI64 v = ((VecI64Object *)o)->vec; + if ((size_t)i < (size_t)v.len) { + return PyLong_FromLongLong(v.buf->items[i]); + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + return PyLong_FromLongLong(v.buf->items[i + v.len]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { + if (start < 0) + start += vec.len; + if (end < 0) + end += vec.len; + if (start < 0) + start = 0; + if (start >= vec.len) + start = vec.len; + if (end < start) + end = start; + if (end > vec.len) + end = vec.len; + int64_t slicelength = end - start; + VecI64 res = vec_alloc(slicelength); + if (VEC_IS_ERROR(res)) + return res; + res.len = slicelength; + for (Py_ssize_t i = 0; i < slicelength; i++) + res.buf->items[i] = vec.buf->items[start + i]; + return res; +} + +PyObject *vec_subscript(PyObject *self, PyObject *item) { + VecI64 vec = ((VecI64Object *)self)->vec; + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if ((size_t)i < (size_t)vec.len) { + return PyLong_FromLongLong(vec.buf->items[i]); + } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { + return PyLong_FromLongLong(vec.buf->items[i + vec.len]); + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } + } else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step; + if (PySlice_Unpack(item, &start, &stop, &step) < 0) + return NULL; + Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); + VecI64 res = vec_alloc(slicelength); + if (VEC_IS_ERROR(res)) + return NULL; + res.len = slicelength; + Py_ssize_t j = start; + for (Py_ssize_t i = 0; i < slicelength; i++) { + res.buf->items[i] = vec.buf->items[j]; + j += step; + } + return VecI64_Box(res); + } else { + PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", + item->ob_type->tp_name); + return NULL; + } +} + +int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + VecI64 v = ((VecI64Object *)self)->vec; + if ((size_t)i < (size_t)v.len) { + long long x = PyLong_AsLongLong(o); + if (x == -1 && PyErr_Occurred()) + return -1; + v.buf->items[i] = x; + return 0; + } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + long long x = PyLong_AsLongLong(o); + if (x == -1 && PyErr_Occurred()) + return -1; + v.buf->items[i + v.len] = x; + return 0; + } else { + PyErr_SetString(PyExc_IndexError, "index out of range"); + return -1; + } +} + +static Py_ssize_t vec_length(PyObject *o) { + // TODO: Type check o + return ((VecI64Object *)o)->vec.len; +} + +static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { + int cmp = 1; + PyObject *res; + if (op == Py_EQ || op == Py_NE) { + if (other->ob_type != &VecI64Type) + cmp = 0; + else { + VecI64 x = ((VecI64Object *)self)->vec; + VecI64 y = ((VecI64Object *)other)->vec; + if (x.len != y.len) { + cmp = 0; + } else { + for (Py_ssize_t i = 0; i < x.len; i++) { + if (x.buf->items[i] != y.buf->items[i]) { + cmp = 0; + break; + } + } + } + } + if (op == Py_NE) + cmp = cmp ^ 1; + res = cmp ? Py_True : Py_False; + } else + res = Py_NotImplemented; + Py_INCREF(res); + return res; +} + +VecI64 VecI64_Append(VecI64 vec, int64_t x) { + if (vec.buf && vec.len < VEC_CAP(vec)) { + vec.buf->items[vec.len] = x; + vec.len++; + return vec; + } else { + Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; + Py_ssize_t new_size = 2 * cap + 1; + VecI64 new = vec_alloc(new_size); + if (VEC_IS_ERROR(new)) + return VecI64_Error(); + new.len = vec.len + 1; + if (vec.len > 0) + memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); + new.buf->items[vec.len] = x; + Py_XDECREF(vec.buf); + return new; + } +} + +VecI64 VecI64_Remove(VecI64 v, int64_t x) { + for (Py_ssize_t i = 0; i < v.len; i++) { + if (v.buf->items[i] == x) { + for (; i < v.len - 1; i++) { + v.buf->items[i] = v.buf->items[i + 1]; + } + v.len--; + VEC_INCREF(v); + return v; + } + } + PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + return VecI64_Error(); +} + +VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { + VecI64PopResult result; + + if (index < 0) + index += v.len; + + if (index < 0 || index >= v.len) { + PyErr_SetString(PyExc_IndexError, "index out of range"); + result.f0 = VecI64_Error(); + result.f1 = 0; + return result; + } + + result.f1 = v.buf->items[index]; + for (Py_ssize_t i = index; i < v.len - 1; i++) { + v.buf->items[i] = v.buf->items[i + 1]; + } + + v.len--; + VEC_INCREF(v); + result.f0 = v; + return result; +} + +static PyMappingMethods VecI64Mapping = { + .mp_length = vec_length, + .mp_subscript = vec_subscript, +}; + +static PySequenceMethods VecI64Sequence = { + .sq_item = vec_get_item, + .sq_ass_item = vec_ass_item, +}; + +static PyMethodDef vec_methods[] = { + {NULL, NULL, 0, NULL}, /* Sentinel */ +}; + +PyTypeObject VecI64BufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vecbuf[i64]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), + .tp_itemsize = sizeof(int64_t), + .tp_flags = Py_TPFLAGS_DEFAULT, + //.tp_new = vecbuf_i64_new, //?? + .tp_free = PyObject_Del, +}; + +PyTypeObject VecI64Type = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "vec[i64]", + .tp_doc = "vec doc", + .tp_basicsize = sizeof(VecI64Object), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = vec_new, + //.tp_free = PyObject_Del, + .tp_dealloc = (destructor)vec_dealloc, + .tp_repr = (reprfunc)vec_repr, + .tp_as_sequence = &VecI64Sequence, + .tp_as_mapping = &VecI64Mapping, + .tp_richcompare = vec_richcompare, + .tp_methods = vec_methods, +}; + +VecI64Features I64Features = { + &VecI64Type, + &VecI64BufType, + VecI64_New, + VecI64_Box, + VecI64_Unbox, + VecI64_ConvertFromNested, + VecI64_Append, + VecI64_Pop, + VecI64_Remove, + VecI64_Slice, +}; From 947010bf5dc0d4b81be78412e2b07b921a840f02 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:24:17 +0000 Subject: [PATCH 125/208] Generalize parts of the template --- mypyc/lib-rt/vecs/vec_template.c | 58 ++++++++++++++++---------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index e0097bb848c14..444192525d266 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -5,8 +5,8 @@ // as vec[i64] or vec[float]. Assume that certain #defines are provided that // provide all the item type specific definitions: // -// PREFIX type name prefix used in definitions (e.g. VecI64) -// VECTYPE C struct used for the vec (e.g. VecI64) +// PREFIX name prefix used for non-static definitions (e.g. VecI64) +// VEC C struct used for the vec (e.g. VecI64) // BOXTYPE PyTypeObject used for a boxed vec (e.g. VecI64Object) // BUFOBJ C struct used for the buffer object (e.g. VecI64BufObject) // BUFTYPE PyTypeObject used for the buffer object (e.g. VecI64BufType) @@ -25,7 +25,7 @@ static VecI64 vec_alloc(Py_ssize_t size) } else { buf = PyObject_NewVar(VecI64BufObject, &VecI64BufType, size); if (buf == NULL) - return VecI64_Error(); + return PREFIX##Error(); } VecI64 res = { .buf = buf }; return res; @@ -36,7 +36,7 @@ static void vec_dealloc(VecI64Object *self) { PyObject_Del(self); } -PyObject *VecI64_Box(VecI64 vec) { +PyObject *PREFIX##Box(VecI64 vec) { VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); if (obj == NULL) return NULL; @@ -44,7 +44,7 @@ PyObject *VecI64_Box(VecI64 vec) { return (PyObject *)obj; } -VecI64 VecI64_Unbox(PyObject *obj) { +VecI64 PREFIX##Unbox(PyObject *obj) { if (obj->ob_type == &VecI64Type) { VecI64 result = ((VecI64Object *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? @@ -52,15 +52,15 @@ VecI64 VecI64_Unbox(PyObject *obj) { } else { // TODO: Better error message PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); - return VecI64_Error(); + return PREFIX##Error(); } } -VecI64 VecI64_ConvertFromNested(VecNestedBufItem item) { +VecI64 PREFIX##ConvertFromNested(VecNestedBufItem item) { return (VecI64) { item.len, (VecI64BufObject *)item.buf }; } -VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { +VecI64 PREFIX##New(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; VecI64 vec = vec_alloc(cap); @@ -73,7 +73,7 @@ VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { return vec; } -PyObject *VecI64_FromIterable(PyObject *iterable) { +PyObject *PREFIX##FromIterable(PyObject *iterable) { VecI64 v = vec_alloc(0); if (VEC_IS_ERROR(v)) return NULL; @@ -93,7 +93,7 @@ PyObject *VecI64_FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - v = VecI64_Append(v, x); + v = PREFIX##Append(v, x); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); VEC_DECREF(v); @@ -105,7 +105,7 @@ PyObject *VecI64_FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - return VecI64_Box(v); + return PREFIX##Box(v); } PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { @@ -115,9 +115,9 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return NULL; } if (init == NULL) { - return VecI64_Box(VecI64_New(0, 0)); + return PREFIX##Box(PREFIX##New(0, 0)); } else { - return (PyObject *)VecI64_FromIterable(init); + return (PyObject *)PREFIX##FromIterable(init); } } @@ -137,7 +137,7 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { +VecI64 PREFIX##Slice(VecI64 vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -188,7 +188,7 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = vec.buf->items[j]; j += step; } - return VecI64_Box(res); + return PREFIX##Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -252,7 +252,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecI64 VecI64_Append(VecI64 vec, int64_t x) { +VecI64 PREFIX##Append(VecI64 vec, int64_t x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; @@ -262,7 +262,7 @@ VecI64 VecI64_Append(VecI64 vec, int64_t x) { Py_ssize_t new_size = 2 * cap + 1; VecI64 new = vec_alloc(new_size); if (VEC_IS_ERROR(new)) - return VecI64_Error(); + return PREFIX##Error(); new.len = vec.len + 1; if (vec.len > 0) memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); @@ -272,7 +272,7 @@ VecI64 VecI64_Append(VecI64 vec, int64_t x) { } } -VecI64 VecI64_Remove(VecI64 v, int64_t x) { +VecI64 PREFIX##Remove(VecI64 v, int64_t x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -284,10 +284,10 @@ VecI64 VecI64_Remove(VecI64 v, int64_t x) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return VecI64_Error(); + return PREFIX##Error(); } -VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { +VecI64PopResult PREFIX##Pop(VecI64 v, Py_ssize_t index) { VecI64PopResult result; if (index < 0) @@ -295,7 +295,7 @@ VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = VecI64_Error(); + result.f0 = PREFIX##Error(); result.f1 = 0; return result; } @@ -356,12 +356,12 @@ PyTypeObject VecI64Type = { VecI64Features I64Features = { &VecI64Type, &VecI64BufType, - VecI64_New, - VecI64_Box, - VecI64_Unbox, - VecI64_ConvertFromNested, - VecI64_Append, - VecI64_Pop, - VecI64_Remove, - VecI64_Slice, + PREFIX##New, + PREFIX##Box, + PREFIX##Unbox, + PREFIX##ConvertFromNested, + PREFIX##Append, + PREFIX##Pop, + PREFIX##Remove, + PREFIX##Slice, }; From bde9dac1dbfd5a0b26fa2ded2429ca69e49aa0ac Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:28:40 +0000 Subject: [PATCH 126/208] Generalize more --- mypyc/lib-rt/vecs/vec_template.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 444192525d266..26ff3af12dbd1 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -7,23 +7,24 @@ // // PREFIX name prefix used for non-static definitions (e.g. VecI64) // VEC C struct used for the vec (e.g. VecI64) -// BOXTYPE PyTypeObject used for a boxed vec (e.g. VecI64Object) -// BUFOBJ C struct used for the buffer object (e.g. VecI64BufObject) -// BUFTYPE PyTypeObject used for the buffer object (e.g. VecI64BufType) #define PY_SSIZE_T_CLEAN #include #include "vecs.h" +#define VEC_TYPE PREFIX##Type +#define BUF_OBJECT PREFIX##BufObject +#define BUF_TYPE PREFIX##BufType + // Alloc a partially initialized vec. Caller *must* initialize len. static VecI64 vec_alloc(Py_ssize_t size) { - VecI64BufObject *buf; + BUF_OBJECT *buf; /* TODO: Check for overflow */ if (size == 0) { buf = NULL; } else { - buf = PyObject_NewVar(VecI64BufObject, &VecI64BufType, size); + buf = PyObject_NewVar(BUF_OBJECT, &BUF_TYPE, size); if (buf == NULL) return PREFIX##Error(); } @@ -37,7 +38,7 @@ static void vec_dealloc(VecI64Object *self) { } PyObject *PREFIX##Box(VecI64 vec) { - VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); + VecI64Object *obj = PyObject_New(VecI64Object, &VEC_TYPE); if (obj == NULL) return NULL; obj->vec = vec; @@ -45,7 +46,7 @@ PyObject *PREFIX##Box(VecI64 vec) { } VecI64 PREFIX##Unbox(PyObject *obj) { - if (obj->ob_type == &VecI64Type) { + if (obj->ob_type == &VEC_TYPE) { VecI64 result = ((VecI64Object *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? return result; @@ -57,7 +58,7 @@ VecI64 PREFIX##Unbox(PyObject *obj) { } VecI64 PREFIX##ConvertFromNested(VecNestedBufItem item) { - return (VecI64) { item.len, (VecI64BufObject *)item.buf }; + return (VecI64) { item.len, (BUF_OBJECT *)item.buf }; } VecI64 PREFIX##New(Py_ssize_t size, Py_ssize_t cap) { @@ -227,7 +228,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { int cmp = 1; PyObject *res; if (op == Py_EQ || op == Py_NE) { - if (other->ob_type != &VecI64Type) + if (other->ob_type != &VEC_TYPE) cmp = 0; else { VecI64 x = ((VecI64Object *)self)->vec; @@ -325,7 +326,7 @@ static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecI64BufType = { +PyTypeObject BUF_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf[i64]", .tp_doc = "vec doc", @@ -336,7 +337,7 @@ PyTypeObject VecI64BufType = { .tp_free = PyObject_Del, }; -PyTypeObject VecI64Type = { +PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec[i64]", .tp_doc = "vec doc", @@ -354,8 +355,8 @@ PyTypeObject VecI64Type = { }; VecI64Features I64Features = { - &VecI64Type, - &VecI64BufType, + &VEC_TYPE, + &BUF_TYPE, PREFIX##New, PREFIX##Box, PREFIX##Unbox, From 3c2e1ad84fd8967e57900b9d9e8c861c8fbd04c6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:29:58 +0000 Subject: [PATCH 127/208] Generalize more --- mypyc/lib-rt/vecs/vec_template.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 26ff3af12dbd1..840b4e290698b 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -13,6 +13,7 @@ #include "vecs.h" #define VEC_TYPE PREFIX##Type +#define VEC_OBJECT PREFIX##Object #define BUF_OBJECT PREFIX##BufObject #define BUF_TYPE PREFIX##BufType @@ -32,13 +33,13 @@ static VecI64 vec_alloc(Py_ssize_t size) return res; } -static void vec_dealloc(VecI64Object *self) { +static void vec_dealloc(VEC_OBJECT *self) { Py_CLEAR(self->vec.buf); PyObject_Del(self); } PyObject *PREFIX##Box(VecI64 vec) { - VecI64Object *obj = PyObject_New(VecI64Object, &VEC_TYPE); + VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); if (obj == NULL) return NULL; obj->vec = vec; @@ -47,7 +48,7 @@ PyObject *PREFIX##Box(VecI64 vec) { VecI64 PREFIX##Unbox(PyObject *obj) { if (obj->ob_type == &VEC_TYPE) { - VecI64 result = ((VecI64Object *)obj)->vec; + VecI64 result = ((VEC_OBJECT *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? return result; } else { @@ -127,7 +128,7 @@ PyObject *vec_repr(PyObject *self) { } PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecI64 v = ((VecI64Object *)o)->vec; + VecI64 v = ((VEC_OBJECT *)o)->vec; if ((size_t)i < (size_t)v.len) { return PyLong_FromLongLong(v.buf->items[i]); } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { @@ -162,7 +163,7 @@ VecI64 PREFIX##Slice(VecI64 vec, int64_t start, int64_t end) { } PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecI64 vec = ((VecI64Object *)self)->vec; + VecI64 vec = ((VEC_OBJECT *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -200,7 +201,7 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (Vec_CheckFloatError(o)) return -1; - VecI64 v = ((VecI64Object *)self)->vec; + VecI64 v = ((VEC_OBJECT *)self)->vec; if ((size_t)i < (size_t)v.len) { long long x = PyLong_AsLongLong(o); if (x == -1 && PyErr_Occurred()) @@ -221,7 +222,7 @@ int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { static Py_ssize_t vec_length(PyObject *o) { // TODO: Type check o - return ((VecI64Object *)o)->vec.len; + return ((VEC_OBJECT *)o)->vec.len; } static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { @@ -231,8 +232,8 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { if (other->ob_type != &VEC_TYPE) cmp = 0; else { - VecI64 x = ((VecI64Object *)self)->vec; - VecI64 y = ((VecI64Object *)other)->vec; + VecI64 x = ((VEC_OBJECT *)self)->vec; + VecI64 y = ((VEC_OBJECT *)other)->vec; if (x.len != y.len) { cmp = 0; } else { @@ -330,7 +331,7 @@ PyTypeObject BUF_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf[i64]", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), + .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(int64_t), .tp_itemsize = sizeof(int64_t), .tp_flags = Py_TPFLAGS_DEFAULT, //.tp_new = vecbuf_i64_new, //?? @@ -341,7 +342,7 @@ PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec[i64]", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object), + .tp_basicsize = sizeof(VEC_OBJECT), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_new = vec_new, From 361d947ac97e0d5ffed9e6fb6e719f8b8ca2ceb9 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:32:45 +0000 Subject: [PATCH 128/208] Generalize more --- mypyc/lib-rt/vecs/vec_template.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 840b4e290698b..2654efa7f2d27 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -289,8 +289,8 @@ VecI64 PREFIX##Remove(VecI64 v, int64_t x) { return PREFIX##Error(); } -VecI64PopResult PREFIX##Pop(VecI64 v, Py_ssize_t index) { - VecI64PopResult result; +PREFIX##PopResult PREFIX##Pop(VecI64 v, Py_ssize_t index) { + PREFIX##PopResult result; if (index < 0) index += v.len; @@ -313,12 +313,12 @@ VecI64PopResult PREFIX##Pop(VecI64 v, Py_ssize_t index) { return result; } -static PyMappingMethods VecI64Mapping = { +static PyMappingMethods vec_mapping_methods = { .mp_length = vec_length, .mp_subscript = vec_subscript, }; -static PySequenceMethods VecI64Sequence = { +static PySequenceMethods vec_sequence_methods = { .sq_item = vec_get_item, .sq_ass_item = vec_ass_item, }; @@ -334,7 +334,7 @@ PyTypeObject BUF_TYPE = { .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(int64_t), .tp_itemsize = sizeof(int64_t), .tp_flags = Py_TPFLAGS_DEFAULT, - //.tp_new = vecbuf_i64_new, //?? + //.tp_new = ?? .tp_free = PyObject_Del, }; @@ -349,13 +349,13 @@ PyTypeObject VEC_TYPE = { //.tp_free = PyObject_Del, .tp_dealloc = (destructor)vec_dealloc, .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecI64Sequence, - .tp_as_mapping = &VecI64Mapping, + .tp_as_sequence = &vec_sequence_methods, + .tp_as_mapping = &vec_mapping_methods, .tp_richcompare = vec_richcompare, .tp_methods = vec_methods, }; -VecI64Features I64Features = { +PREFIX##Features I64Features = { &VEC_TYPE, &BUF_TYPE, PREFIX##New, From f31c0b91a71475af965c4bcc17c14fd3e5ac6597 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:53:06 +0000 Subject: [PATCH 129/208] Do it properly --- mypyc/lib-rt/vecs/vec_i64.c | 364 +------------------------------ mypyc/lib-rt/vecs/vec_template.c | 72 +++--- 2 files changed, 45 insertions(+), 391 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 33fa27b4aae80..750a535ce27a7 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -1,354 +1,10 @@ -#define PY_SSIZE_T_CLEAN -#include -#include "vecs.h" - -// Alloc a partially initialized vec. Caller *must* initialize len. -static VecI64 vec_alloc(Py_ssize_t size) -{ - VecI64BufObject *buf; - /* TODO: Check for overflow */ - if (size == 0) { - buf = NULL; - } else { - buf = PyObject_NewVar(VecI64BufObject, &VecI64BufType, size); - if (buf == NULL) - return VecI64_Error(); - } - VecI64 res = { .buf = buf }; - return res; -} - -static void vec_dealloc(VecI64Object *self) { - Py_CLEAR(self->vec.buf); - PyObject_Del(self); -} - -PyObject *VecI64_Box(VecI64 vec) { - VecI64Object *obj = PyObject_New(VecI64Object, &VecI64Type); - if (obj == NULL) - return NULL; - obj->vec = vec; - return (PyObject *)obj; -} - -VecI64 VecI64_Unbox(PyObject *obj) { - if (obj->ob_type == &VecI64Type) { - VecI64 result = ((VecI64Object *)obj)->vec; - VEC_INCREF(result); // TODO: Should we borrow instead? - return result; - } else { - // TODO: Better error message - PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); - return VecI64_Error(); - } -} - -VecI64 VecI64_ConvertFromNested(VecNestedBufItem item) { - return (VecI64) { item.len, (VecI64BufObject *)item.buf }; -} - -VecI64 VecI64_New(Py_ssize_t size, Py_ssize_t cap) { - if (cap < size) - size = cap; - VecI64 vec = vec_alloc(cap); - if (VEC_IS_ERROR(vec)) - return vec; - for (Py_ssize_t i = 0; i < cap; i++) { - vec.buf->items[i] = 0; - } - vec.len = size; - return vec; -} - -PyObject *VecI64_FromIterable(PyObject *iterable) { - VecI64 v = vec_alloc(0); - if (VEC_IS_ERROR(v)) - return NULL; - v.len = 0; - - PyObject *iter = PyObject_GetIter(iterable); - if (iter == NULL) { - VEC_DECREF(v); - return NULL; - } - PyObject *item; - while ((item = PyIter_Next(iter)) != NULL) { - int64_t x = PyLong_AsLongLong(item); - Py_DECREF(item); - if (x == -1 && PyErr_Occurred()) { - Py_DECREF(iter); - VEC_DECREF(v); - return NULL; - } - v = VecI64_Append(v, x); - if (VEC_IS_ERROR(v)) { - Py_DECREF(iter); - VEC_DECREF(v); - return NULL; - } - } - Py_DECREF(iter); - if (PyErr_Occurred()) { - VEC_DECREF(v); - return NULL; - } - return VecI64_Box(v); -} - -PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {"", NULL}; - PyObject *init = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { - return NULL; - } - if (init == NULL) { - return VecI64_Box(VecI64_New(0, 0)); - } else { - return (PyObject *)VecI64_FromIterable(init); - } -} - -PyObject *vec_repr(PyObject *self) { - return Vec_GenericRepr(self, (size_t)I64TypeObj, 0, 1); -} - -PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecI64 v = ((VecI64Object *)o)->vec; - if ((size_t)i < (size_t)v.len) { - return PyLong_FromLongLong(v.buf->items[i]); - } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - return PyLong_FromLongLong(v.buf->items[i + v.len]); - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } -} - -VecI64 VecI64_Slice(VecI64 vec, int64_t start, int64_t end) { - if (start < 0) - start += vec.len; - if (end < 0) - end += vec.len; - if (start < 0) - start = 0; - if (start >= vec.len) - start = vec.len; - if (end < start) - end = start; - if (end > vec.len) - end = vec.len; - int64_t slicelength = end - start; - VecI64 res = vec_alloc(slicelength); - if (VEC_IS_ERROR(res)) - return res; - res.len = slicelength; - for (Py_ssize_t i = 0; i < slicelength; i++) - res.buf->items[i] = vec.buf->items[start + i]; - return res; -} - -PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecI64 vec = ((VecI64Object *)self)->vec; - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) - return NULL; - if ((size_t)i < (size_t)vec.len) { - return PyLong_FromLongLong(vec.buf->items[i]); - } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { - return PyLong_FromLongLong(vec.buf->items[i + vec.len]); - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } - } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) - return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecI64 res = vec_alloc(slicelength); - if (VEC_IS_ERROR(res)) - return NULL; - res.len = slicelength; - Py_ssize_t j = start; - for (Py_ssize_t i = 0; i < slicelength; i++) { - res.buf->items[i] = vec.buf->items[j]; - j += step; - } - return VecI64_Box(res); - } else { - PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", - item->ob_type->tp_name); - return NULL; - } -} - -int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - if (Vec_CheckFloatError(o)) - return -1; - VecI64 v = ((VecI64Object *)self)->vec; - if ((size_t)i < (size_t)v.len) { - long long x = PyLong_AsLongLong(o); - if (x == -1 && PyErr_Occurred()) - return -1; - v.buf->items[i] = x; - return 0; - } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - long long x = PyLong_AsLongLong(o); - if (x == -1 && PyErr_Occurred()) - return -1; - v.buf->items[i + v.len] = x; - return 0; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return -1; - } -} - -static Py_ssize_t vec_length(PyObject *o) { - // TODO: Type check o - return ((VecI64Object *)o)->vec.len; -} - -static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { - int cmp = 1; - PyObject *res; - if (op == Py_EQ || op == Py_NE) { - if (other->ob_type != &VecI64Type) - cmp = 0; - else { - VecI64 x = ((VecI64Object *)self)->vec; - VecI64 y = ((VecI64Object *)other)->vec; - if (x.len != y.len) { - cmp = 0; - } else { - for (Py_ssize_t i = 0; i < x.len; i++) { - if (x.buf->items[i] != y.buf->items[i]) { - cmp = 0; - break; - } - } - } - } - if (op == Py_NE) - cmp = cmp ^ 1; - res = cmp ? Py_True : Py_False; - } else - res = Py_NotImplemented; - Py_INCREF(res); - return res; -} - -VecI64 VecI64_Append(VecI64 vec, int64_t x) { - if (vec.buf && vec.len < VEC_CAP(vec)) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; - Py_ssize_t new_size = 2 * cap + 1; - VecI64 new = vec_alloc(new_size); - if (VEC_IS_ERROR(new)) - return VecI64_Error(); - new.len = vec.len + 1; - if (vec.len > 0) - memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); - new.buf->items[vec.len] = x; - Py_XDECREF(vec.buf); - return new; - } -} - -VecI64 VecI64_Remove(VecI64 v, int64_t x) { - for (Py_ssize_t i = 0; i < v.len; i++) { - if (v.buf->items[i] == x) { - for (; i < v.len - 1; i++) { - v.buf->items[i] = v.buf->items[i + 1]; - } - v.len--; - VEC_INCREF(v); - return v; - } - } - PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return VecI64_Error(); -} - -VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index) { - VecI64PopResult result; - - if (index < 0) - index += v.len; - - if (index < 0 || index >= v.len) { - PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = VecI64_Error(); - result.f1 = 0; - return result; - } - - result.f1 = v.buf->items[index]; - for (Py_ssize_t i = index; i < v.len - 1; i++) { - v.buf->items[i] = v.buf->items[i + 1]; - } - - v.len--; - VEC_INCREF(v); - result.f0 = v; - return result; -} - -static PyMappingMethods VecI64Mapping = { - .mp_length = vec_length, - .mp_subscript = vec_subscript, -}; - -static PySequenceMethods VecI64Sequence = { - .sq_item = vec_get_item, - .sq_ass_item = vec_ass_item, -}; - -static PyMethodDef vec_methods[] = { - {NULL, NULL, 0, NULL}, /* Sentinel */ -}; - -PyTypeObject VecI64BufType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecbuf[i64]", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object) - sizeof(int64_t), - .tp_itemsize = sizeof(int64_t), - .tp_flags = Py_TPFLAGS_DEFAULT, - //.tp_new = vecbuf_i64_new, //?? - .tp_free = PyObject_Del, -}; - -PyTypeObject VecI64Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec[i64]", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecI64Object), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = vec_new, - //.tp_free = PyObject_Del, - .tp_dealloc = (destructor)vec_dealloc, - .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecI64Sequence, - .tp_as_mapping = &VecI64Mapping, - .tp_richcompare = vec_richcompare, - .tp_methods = vec_methods, -}; - -VecI64Features I64Features = { - &VecI64Type, - &VecI64BufType, - VecI64_New, - VecI64_Box, - VecI64_Unbox, - VecI64_ConvertFromNested, - VecI64_Append, - VecI64_Pop, - VecI64_Remove, - VecI64_Slice, -}; +#define PREFIX VecI64 +#define VEC_TYPE VecI64Type +#define VEC_OBJECT VecI64Object +#define BUF_OBJECT VecI64BufObject +#define BUF_TYPE VecI64BufType +#define NAME(suffix) VecI64##suffix +#define FUNC(suffix) VecI64_##suffix +#define ITEM_STR "i64" + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 2654efa7f2d27..522a3f60dbdd7 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -6,17 +6,15 @@ // provide all the item type specific definitions: // // PREFIX name prefix used for non-static definitions (e.g. VecI64) -// VEC C struct used for the vec (e.g. VecI64) + +#ifndef PREFIX +#error "PREFIX must be defined" +#endif #define PY_SSIZE_T_CLEAN #include #include "vecs.h" -#define VEC_TYPE PREFIX##Type -#define VEC_OBJECT PREFIX##Object -#define BUF_OBJECT PREFIX##BufObject -#define BUF_TYPE PREFIX##BufType - // Alloc a partially initialized vec. Caller *must* initialize len. static VecI64 vec_alloc(Py_ssize_t size) { @@ -27,7 +25,7 @@ static VecI64 vec_alloc(Py_ssize_t size) } else { buf = PyObject_NewVar(BUF_OBJECT, &BUF_TYPE, size); if (buf == NULL) - return PREFIX##Error(); + return FUNC(Error)(); } VecI64 res = { .buf = buf }; return res; @@ -38,7 +36,7 @@ static void vec_dealloc(VEC_OBJECT *self) { PyObject_Del(self); } -PyObject *PREFIX##Box(VecI64 vec) { +PyObject *FUNC(Box)(VecI64 vec) { VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); if (obj == NULL) return NULL; @@ -46,7 +44,7 @@ PyObject *PREFIX##Box(VecI64 vec) { return (PyObject *)obj; } -VecI64 PREFIX##Unbox(PyObject *obj) { +VecI64 FUNC(Unbox)(PyObject *obj) { if (obj->ob_type == &VEC_TYPE) { VecI64 result = ((VEC_OBJECT *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? @@ -54,15 +52,15 @@ VecI64 PREFIX##Unbox(PyObject *obj) { } else { // TODO: Better error message PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); - return PREFIX##Error(); + return FUNC(Error)(); } } -VecI64 PREFIX##ConvertFromNested(VecNestedBufItem item) { +VecI64 FUNC(ConvertFromNested)(VecNestedBufItem item) { return (VecI64) { item.len, (BUF_OBJECT *)item.buf }; } -VecI64 PREFIX##New(Py_ssize_t size, Py_ssize_t cap) { +VecI64 FUNC(New)(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; VecI64 vec = vec_alloc(cap); @@ -75,7 +73,7 @@ VecI64 PREFIX##New(Py_ssize_t size, Py_ssize_t cap) { return vec; } -PyObject *PREFIX##FromIterable(PyObject *iterable) { +PyObject *FUNC(FromIterable)(PyObject *iterable) { VecI64 v = vec_alloc(0); if (VEC_IS_ERROR(v)) return NULL; @@ -95,7 +93,7 @@ PyObject *PREFIX##FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - v = PREFIX##Append(v, x); + v = FUNC(Append)(v, x); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); VEC_DECREF(v); @@ -107,7 +105,7 @@ PyObject *PREFIX##FromIterable(PyObject *iterable) { VEC_DECREF(v); return NULL; } - return PREFIX##Box(v); + return FUNC(Box)(v); } PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { @@ -117,9 +115,9 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { return NULL; } if (init == NULL) { - return PREFIX##Box(PREFIX##New(0, 0)); + return FUNC(Box)(FUNC(New)(0, 0)); } else { - return (PyObject *)PREFIX##FromIterable(init); + return (PyObject *)FUNC(FromIterable)(init); } } @@ -139,7 +137,7 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -VecI64 PREFIX##Slice(VecI64 vec, int64_t start, int64_t end) { +VecI64 FUNC(Slice)(VecI64 vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -190,7 +188,7 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = vec.buf->items[j]; j += step; } - return PREFIX##Box(res); + return FUNC(Box)(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -254,7 +252,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecI64 PREFIX##Append(VecI64 vec, int64_t x) { +VecI64 FUNC(Append)(VecI64 vec, int64_t x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; @@ -264,7 +262,7 @@ VecI64 PREFIX##Append(VecI64 vec, int64_t x) { Py_ssize_t new_size = 2 * cap + 1; VecI64 new = vec_alloc(new_size); if (VEC_IS_ERROR(new)) - return PREFIX##Error(); + return FUNC(Error)(); new.len = vec.len + 1; if (vec.len > 0) memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); @@ -274,7 +272,7 @@ VecI64 PREFIX##Append(VecI64 vec, int64_t x) { } } -VecI64 PREFIX##Remove(VecI64 v, int64_t x) { +VecI64 FUNC(Remove)(VecI64 v, int64_t x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -286,18 +284,18 @@ VecI64 PREFIX##Remove(VecI64 v, int64_t x) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return PREFIX##Error(); + return FUNC(Error)(); } -PREFIX##PopResult PREFIX##Pop(VecI64 v, Py_ssize_t index) { - PREFIX##PopResult result; +NAME(PopResult) FUNC(Pop)(VecI64 v, Py_ssize_t index) { + NAME(PopResult) result; if (index < 0) index += v.len; if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = PREFIX##Error(); + result.f0 = FUNC(Error)(); result.f1 = 0; return result; } @@ -329,7 +327,7 @@ static PyMethodDef vec_methods[] = { PyTypeObject BUF_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecbuf[i64]", + .tp_name = "vecbuf[" ITEM_STR "]", .tp_doc = "vec doc", .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(int64_t), .tp_itemsize = sizeof(int64_t), @@ -340,7 +338,7 @@ PyTypeObject BUF_TYPE = { PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec[i64]", + .tp_name = "vec[" ITEM_STR "]", .tp_doc = "vec doc", .tp_basicsize = sizeof(VEC_OBJECT), .tp_itemsize = 0, @@ -355,15 +353,15 @@ PyTypeObject VEC_TYPE = { .tp_methods = vec_methods, }; -PREFIX##Features I64Features = { +NAME(Features) I64Features = { &VEC_TYPE, &BUF_TYPE, - PREFIX##New, - PREFIX##Box, - PREFIX##Unbox, - PREFIX##ConvertFromNested, - PREFIX##Append, - PREFIX##Pop, - PREFIX##Remove, - PREFIX##Slice, + FUNC(New), + FUNC(Box), + FUNC(Unbox), + FUNC(ConvertFromNested), + FUNC(Append), + FUNC(Pop), + FUNC(Remove), + FUNC(Slice), }; From d819722851560aa01c6585e07e7f2f1c55394841 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:55:32 +0000 Subject: [PATCH 130/208] Generalize more --- mypyc/lib-rt/vecs/vec_i64.c | 3 ++- mypyc/lib-rt/vecs/vec_template.c | 14 +++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 750a535ce27a7..942f8f416b15a 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -5,6 +5,7 @@ #define BUF_TYPE VecI64BufType #define NAME(suffix) VecI64##suffix #define FUNC(suffix) VecI64_##suffix -#define ITEM_STR "i64" +#define ITEM_TYPE_STR "i64" +#define ITEM_C_TYPE int64_t #include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 522a3f60dbdd7..752fe047ae075 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -252,7 +252,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecI64 FUNC(Append)(VecI64 vec, int64_t x) { +VecI64 FUNC(Append)(VecI64 vec, ITEM_C_TYPE x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; @@ -265,14 +265,14 @@ VecI64 FUNC(Append)(VecI64 vec, int64_t x) { return FUNC(Error)(); new.len = vec.len + 1; if (vec.len > 0) - memcpy(new.buf->items, vec.buf->items, sizeof(int64_t) * vec.len); + memcpy(new.buf->items, vec.buf->items, sizeof(ITEM_C_TYPE) * vec.len); new.buf->items[vec.len] = x; Py_XDECREF(vec.buf); return new; } } -VecI64 FUNC(Remove)(VecI64 v, int64_t x) { +VecI64 FUNC(Remove)(VecI64 v, ITEM_C_TYPE x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -327,10 +327,10 @@ static PyMethodDef vec_methods[] = { PyTypeObject BUF_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecbuf[" ITEM_STR "]", + .tp_name = "vecbuf[" ITEM_TYPE_STR "]", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(int64_t), - .tp_itemsize = sizeof(int64_t), + .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(ITEM_C_TYPE), + .tp_itemsize = sizeof(ITEM_C_TYPE), .tp_flags = Py_TPFLAGS_DEFAULT, //.tp_new = ?? .tp_free = PyObject_Del, @@ -338,7 +338,7 @@ PyTypeObject BUF_TYPE = { PyTypeObject VEC_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec[" ITEM_STR "]", + .tp_name = "vec[" ITEM_TYPE_STR "]", .tp_doc = "vec doc", .tp_basicsize = sizeof(VEC_OBJECT), .tp_itemsize = 0, From 5e25e2d6878fcd88bc82f23df4c4e3d087bfd4b1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 11:59:01 +0000 Subject: [PATCH 131/208] Generalize more, including BOX_ITEM --- mypyc/lib-rt/vecs/vec_i64.c | 3 +++ mypyc/lib-rt/vecs/vec_template.c | 10 +++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 942f8f416b15a..98e36f9e96a9e 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -7,5 +7,8 @@ #define FUNC(suffix) VecI64_##suffix #define ITEM_TYPE_STR "i64" #define ITEM_C_TYPE int64_t +#define FEATURES I64Features + +#define BOX_ITEM(x) PyLong_FromLongLong(x) #include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 752fe047ae075..f290cba0eb5f9 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -128,9 +128,9 @@ PyObject *vec_repr(PyObject *self) { PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { VecI64 v = ((VEC_OBJECT *)o)->vec; if ((size_t)i < (size_t)v.len) { - return PyLong_FromLongLong(v.buf->items[i]); + return BOX_ITEM(v.buf->items[i]); } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - return PyLong_FromLongLong(v.buf->items[i + v.len]); + return BOX_ITEM(v.buf->items[i + v.len]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -167,9 +167,9 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { if (i == -1 && PyErr_Occurred()) return NULL; if ((size_t)i < (size_t)vec.len) { - return PyLong_FromLongLong(vec.buf->items[i]); + return BOX_ITEM(vec.buf->items[i]); } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { - return PyLong_FromLongLong(vec.buf->items[i + vec.len]); + return BOX_ITEM(vec.buf->items[i + vec.len]); } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; @@ -353,7 +353,7 @@ PyTypeObject VEC_TYPE = { .tp_methods = vec_methods, }; -NAME(Features) I64Features = { +NAME(Features) FEATURES = { &VEC_TYPE, &BUF_TYPE, FUNC(New), From 1381ae0ab1e76faeb5bde5ac9bdd36e10c1a4b79 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 12:35:17 +0000 Subject: [PATCH 132/208] Generalize more --- mypyc/lib-rt/vecs/vec_i64.c | 2 ++ mypyc/lib-rt/vecs/vec_template.c | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 98e36f9e96a9e..efa8656a3012e 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -10,5 +10,7 @@ #define FEATURES I64Features #define BOX_ITEM(x) PyLong_FromLongLong(x) +#define UNBOX_ITEM(x) PyLong_AsLongLong(x) +#define IS_UNBOX_ERROR(x) ((x) == -1 && PyErr_Occurred()) #include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index f290cba0eb5f9..d5262d5772f8d 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -86,9 +86,9 @@ PyObject *FUNC(FromIterable)(PyObject *iterable) { } PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { - int64_t x = PyLong_AsLongLong(item); + ITEM_C_TYPE x = UNBOX_ITEM(item); Py_DECREF(item); - if (x == -1 && PyErr_Occurred()) { + if (IS_UNBOX_ERROR(x)) { Py_DECREF(iter); VEC_DECREF(v); return NULL; @@ -201,14 +201,14 @@ int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { return -1; VecI64 v = ((VEC_OBJECT *)self)->vec; if ((size_t)i < (size_t)v.len) { - long long x = PyLong_AsLongLong(o); - if (x == -1 && PyErr_Occurred()) + ITEM_C_TYPE x = UNBOX_ITEM(o); + if (IS_UNBOX_ERROR(x)) return -1; v.buf->items[i] = x; return 0; } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - long long x = PyLong_AsLongLong(o); - if (x == -1 && PyErr_Occurred()) + ITEM_C_TYPE x = UNBOX_ITEM(o); + if (IS_UNBOX_ERROR(x)) return -1; v.buf->items[i + v.len] = x; return 0; From a69a7ee39e453a5bb18dff1ae94a1f96e1b2211d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 13:04:50 +0000 Subject: [PATCH 133/208] Generalize remaining --- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_template.c | 60 ++++++++++++++++---------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index efa8656a3012e..28ad42c7abaab 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -1,4 +1,4 @@ -#define PREFIX VecI64 +#define VEC VecI64 #define VEC_TYPE VecI64Type #define VEC_OBJECT VecI64Object #define BUF_OBJECT VecI64BufObject diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index d5262d5772f8d..357ddb1d13f68 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -5,10 +5,10 @@ // as vec[i64] or vec[float]. Assume that certain #defines are provided that // provide all the item type specific definitions: // -// PREFIX name prefix used for non-static definitions (e.g. VecI64) +// VEC vec C type (e.g. VecI64) -#ifndef PREFIX -#error "PREFIX must be defined" +#ifndef VEC +#error "VEC must be defined" #endif #define PY_SSIZE_T_CLEAN @@ -16,7 +16,7 @@ #include "vecs.h" // Alloc a partially initialized vec. Caller *must* initialize len. -static VecI64 vec_alloc(Py_ssize_t size) +static VEC vec_alloc(Py_ssize_t size) { BUF_OBJECT *buf; /* TODO: Check for overflow */ @@ -27,7 +27,7 @@ static VecI64 vec_alloc(Py_ssize_t size) if (buf == NULL) return FUNC(Error)(); } - VecI64 res = { .buf = buf }; + VEC res = { .buf = buf }; return res; } @@ -36,7 +36,7 @@ static void vec_dealloc(VEC_OBJECT *self) { PyObject_Del(self); } -PyObject *FUNC(Box)(VecI64 vec) { +PyObject *FUNC(Box)(VEC vec) { VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); if (obj == NULL) return NULL; @@ -44,9 +44,9 @@ PyObject *FUNC(Box)(VecI64 vec) { return (PyObject *)obj; } -VecI64 FUNC(Unbox)(PyObject *obj) { +VEC FUNC(Unbox)(PyObject *obj) { if (obj->ob_type == &VEC_TYPE) { - VecI64 result = ((VEC_OBJECT *)obj)->vec; + VEC result = ((VEC_OBJECT *)obj)->vec; VEC_INCREF(result); // TODO: Should we borrow instead? return result; } else { @@ -56,14 +56,14 @@ VecI64 FUNC(Unbox)(PyObject *obj) { } } -VecI64 FUNC(ConvertFromNested)(VecNestedBufItem item) { - return (VecI64) { item.len, (BUF_OBJECT *)item.buf }; +VEC FUNC(ConvertFromNested)(VecNestedBufItem item) { + return (VEC) { item.len, (BUF_OBJECT *)item.buf }; } -VecI64 FUNC(New)(Py_ssize_t size, Py_ssize_t cap) { +VEC FUNC(New)(Py_ssize_t size, Py_ssize_t cap) { if (cap < size) size = cap; - VecI64 vec = vec_alloc(cap); + VEC vec = vec_alloc(cap); if (VEC_IS_ERROR(vec)) return vec; for (Py_ssize_t i = 0; i < cap; i++) { @@ -74,7 +74,7 @@ VecI64 FUNC(New)(Py_ssize_t size, Py_ssize_t cap) { } PyObject *FUNC(FromIterable)(PyObject *iterable) { - VecI64 v = vec_alloc(0); + VEC v = vec_alloc(0); if (VEC_IS_ERROR(v)) return NULL; v.len = 0; @@ -108,7 +108,7 @@ PyObject *FUNC(FromIterable)(PyObject *iterable) { return FUNC(Box)(v); } -PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { +static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { static char *kwlist[] = {"", NULL}; PyObject *init = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { @@ -121,12 +121,12 @@ PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } } -PyObject *vec_repr(PyObject *self) { +static PyObject *vec_repr(PyObject *self) { return Vec_GenericRepr(self, (size_t)I64TypeObj, 0, 1); } -PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecI64 v = ((VEC_OBJECT *)o)->vec; +static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { + VEC v = ((VEC_OBJECT *)o)->vec; if ((size_t)i < (size_t)v.len) { return BOX_ITEM(v.buf->items[i]); } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { @@ -137,7 +137,7 @@ PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -VecI64 FUNC(Slice)(VecI64 vec, int64_t start, int64_t end) { +VEC FUNC(Slice)(VEC vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -151,7 +151,7 @@ VecI64 FUNC(Slice)(VecI64 vec, int64_t start, int64_t end) { if (end > vec.len) end = vec.len; int64_t slicelength = end - start; - VecI64 res = vec_alloc(slicelength); + VEC res = vec_alloc(slicelength); if (VEC_IS_ERROR(res)) return res; res.len = slicelength; @@ -160,8 +160,8 @@ VecI64 FUNC(Slice)(VecI64 vec, int64_t start, int64_t end) { return res; } -PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecI64 vec = ((VEC_OBJECT *)self)->vec; +static PyObject *vec_subscript(PyObject *self, PyObject *item) { + VEC vec = ((VEC_OBJECT *)self)->vec; if (PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) @@ -179,7 +179,7 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { if (PySlice_Unpack(item, &start, &stop, &step) < 0) return NULL; Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecI64 res = vec_alloc(slicelength); + VEC res = vec_alloc(slicelength); if (VEC_IS_ERROR(res)) return NULL; res.len = slicelength; @@ -196,10 +196,10 @@ PyObject *vec_subscript(PyObject *self, PyObject *item) { } } -int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { +static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (Vec_CheckFloatError(o)) return -1; - VecI64 v = ((VEC_OBJECT *)self)->vec; + VEC v = ((VEC_OBJECT *)self)->vec; if ((size_t)i < (size_t)v.len) { ITEM_C_TYPE x = UNBOX_ITEM(o); if (IS_UNBOX_ERROR(x)) @@ -230,8 +230,8 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { if (other->ob_type != &VEC_TYPE) cmp = 0; else { - VecI64 x = ((VEC_OBJECT *)self)->vec; - VecI64 y = ((VEC_OBJECT *)other)->vec; + VEC x = ((VEC_OBJECT *)self)->vec; + VEC y = ((VEC_OBJECT *)other)->vec; if (x.len != y.len) { cmp = 0; } else { @@ -252,7 +252,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } -VecI64 FUNC(Append)(VecI64 vec, ITEM_C_TYPE x) { +VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; vec.len++; @@ -260,7 +260,7 @@ VecI64 FUNC(Append)(VecI64 vec, ITEM_C_TYPE x) { } else { Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; Py_ssize_t new_size = 2 * cap + 1; - VecI64 new = vec_alloc(new_size); + VEC new = vec_alloc(new_size); if (VEC_IS_ERROR(new)) return FUNC(Error)(); new.len = vec.len + 1; @@ -272,7 +272,7 @@ VecI64 FUNC(Append)(VecI64 vec, ITEM_C_TYPE x) { } } -VecI64 FUNC(Remove)(VecI64 v, ITEM_C_TYPE x) { +VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { for (; i < v.len - 1; i++) { @@ -287,7 +287,7 @@ VecI64 FUNC(Remove)(VecI64 v, ITEM_C_TYPE x) { return FUNC(Error)(); } -NAME(PopResult) FUNC(Pop)(VecI64 v, Py_ssize_t index) { +NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { NAME(PopResult) result; if (index < 0) From 436c4ed5788624142f7fb488600b848edea02368 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 15:58:22 +0000 Subject: [PATCH 134/208] Generalize more --- mypyc/lib-rt/vecs/vec_i64.c | 1 + mypyc/lib-rt/vecs/vec_template.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 28ad42c7abaab..cd7a0e20fa377 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -6,6 +6,7 @@ #define NAME(suffix) VecI64##suffix #define FUNC(suffix) VecI64_##suffix #define ITEM_TYPE_STR "i64" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I64 #define ITEM_C_TYPE int64_t #define FEATURES I64Features diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 357ddb1d13f68..31a0bfa32149c 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -122,7 +122,7 @@ static PyObject *vec_new(PyTypeObject *self, PyObject *args, PyObject *kw) { } static PyObject *vec_repr(PyObject *self) { - return Vec_GenericRepr(self, (size_t)I64TypeObj, 0, 1); + return Vec_GenericRepr(self, ITEM_TYPE_MAGIC, 0, 1); } static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { From 5ee4f4db9f1798a0f5efca6a735285f77ebdf7fa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 15:59:57 +0000 Subject: [PATCH 135/208] Implement vec[float] --- mypyc/lib-rt/vecs/vec_float.c | 17 ++++++++ mypyc/lib-rt/vecs/vecs.c | 49 ++++++++++++++++++++- mypyc/lib-rt/vecs/vecs.h | 82 +++++++++++++++++++++++++++++++++-- 3 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_float.c diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c new file mode 100644 index 0000000000000..126bff8a790d5 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -0,0 +1,17 @@ +#define VEC VecFloat +#define VEC_TYPE VecFloatType +#define VEC_OBJECT VecFloatObject +#define BUF_OBJECT VecFloatBufObject +#define BUF_TYPE VecFloatBufType +#define NAME(suffix) VecFloat##suffix +#define FUNC(suffix) VecFloat_##suffix +#define ITEM_TYPE_STR "float" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_FLOAT +#define ITEM_C_TYPE double +#define FEATURES FloatFeatures + +#define BOX_ITEM(x) PyFloat_FromDouble(x) +#define UNBOX_ITEM(x) PyFloat_AsDouble(x) +#define IS_UNBOX_ERROR(x) ((x) == -1.0 && PyErr_Occurred()) + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 74bce9bcc20a6..ef2c113572c3c 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -70,6 +70,8 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { if (depth == 0) if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { item = PyUnicode_FromFormat("i64"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_FLOAT) { + item = PyUnicode_FromFormat("float"); } else { item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); if (item_type & 1) { @@ -145,6 +147,9 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (item == (PyObject *)I64TypeObj) { Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; + } else if (item == (PyObject *)&PyFloat_Type) { + Py_INCREF(&VecFloatType); + return (PyObject *)&VecFloatType; } else { size_t depth = 0; size_t item_type; @@ -174,6 +179,10 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) item_type = VEC_ITEM_TYPE_I64; depth = 1; // TODO: Check optionals? + } else if (item == (PyObject *)&VecFloatType) { + item_type = VEC_ITEM_TYPE_FLOAT; + depth = 1; + // TODO: Check optionals? } else { item_type = (size_t)item; } @@ -354,6 +363,17 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI64_Box(v); + } else if (VecFloat_Check(vec)) { + double x = PyFloat_AsDouble(item); + if (x == -1 && PyErr_Occurred()) { + return NULL; + } + VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); + v = VecFloat_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecFloat_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -401,6 +421,17 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI64_Box(v); + } else if (VecFloat_Check(vec)) { + double x = PyFloat_AsDouble(item); + if (x == -1 && PyErr_Occurred()) { + return NULL; + } + VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); + v = VecFloat_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecFloat_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -449,6 +480,17 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if (result_item0 == NULL) return NULL; result_item1 = PyLong_FromLongLong(r.f1); + } else if (VecFloat_Check(vec)) { + VecFloat v = ((VecFloatObject *)vec)->vec; + VecFloatPopResult r; + r = VecFloat_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + + result_item0 = VecFloat_Box(r.f0); + if (result_item0 == NULL) + return NULL; + result_item1 = PyFloat_FromDouble(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; @@ -520,9 +562,10 @@ static PyModuleDef vecsmodule = { }; static VecCapsule Capsule = { - &I64Features, &TFeatures, &TExtFeatures, + &I64Features, + &FloatFeatures, }; PyMODINIT_FUNC @@ -558,6 +601,10 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecI64BufType) < 0) return NULL; + if (PyType_Ready(&VecFloatType) < 0) + return NULL; + if (PyType_Ready(&VecFloatBufType) < 0) + return NULL; PyObject *m = PyModule_Create(&vecsmodule); if (m == NULL) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index c40cb88a81cd4..187ac1fc60434 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -10,6 +10,7 @@ // Item type constants; must be even but not multiples of 4 (2 + 4 * n) #define VEC_ITEM_TYPE_I64 2 +#define VEC_ITEM_TYPE_FLOAT 18 inline size_t Vec_IsMagicItemType(size_t item_type) { return item_type & 2; @@ -25,6 +26,12 @@ typedef struct _VecI64BufObject { int64_t items[1]; } VecI64BufObject; +// vecbuf[float] +typedef struct _VecFloatBufObject { + PyObject_VAR_HEAD + double items[1]; +} VecFloatBufObject; + // Simple generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecTBufObject { PyObject_VAR_HEAD @@ -58,6 +65,11 @@ typedef struct _VecI64 { VecI64BufObject *buf; } VecI64; +typedef struct _VecFloat { + Py_ssize_t len; + VecFloatBufObject *buf; +} VecFloat; + typedef struct _VecT { Py_ssize_t len; VecTBufObject *buf; @@ -84,6 +96,12 @@ typedef struct _VecI64Object { VecI64 vec; } VecI64Object; +// Boxed vec[float] +typedef struct _VecFloatObject { + PyObject_HEAD + VecFloat vec; +} VecFloatObject; + // Simple boxed generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecTObject { PyObject_HEAD @@ -106,7 +124,17 @@ typedef struct tuple_T2V88 { static tuple_T2V88 tuple_undefined_T2V88 = { { -1, NULL } , 0 }; #endif +#ifndef MYPYC_DECLARED_tuple_T2VFF +#define MYPYC_DECLARED_tuple_T2VFF +typedef struct tuple_T2VFF { + VecFloat f0; + double f1; +} tuple_T2VFF; +static tuple_T2VFF tuple_undefined_T2VFF = { { -1, NULL } , 0.0 }; +#endif + typedef tuple_T2V88 VecI64PopResult; +typedef tuple_T2VFF VecFloatPopResult; // vec[i64] operations + type objects (stored in a capsule) typedef struct _VecI64Features { @@ -127,6 +155,25 @@ typedef struct _VecI64Features { // iter? } VecI64Features; +// vec[float] operations + type objects (stored in a capsule) +typedef struct _VecFloatFeatures { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecFloat (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecFloat); + VecFloat (*unbox)(PyObject *); + VecFloat (*convert_from_nested)(VecNestedBufItem); + VecFloat (*append)(VecFloat, double); + VecFloatPopResult (*pop)(VecFloat, Py_ssize_t); + VecFloat (*remove)(VecFloat, double); + // TODO: Py_ssize_t + VecFloat (*slice)(VecFloat, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, double); + // iter? +} VecFloatFeatures; + #ifndef MYPYC_DECLARED_tuple_T2VOO #define MYPYC_DECLARED_tuple_T2VOO typedef struct tuple_T2VOO { @@ -191,9 +238,10 @@ typedef struct _VecNestedFeatures { } VecNestedFeatures; typedef struct { - VecI64Features *i64; VecTFeatures *t; VecNestedFeatures *nested; + VecI64Features *i64; + VecFloatFeatures *float_; } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) @@ -209,6 +257,11 @@ inline VecI64 VecI64_Error() { return v; } +inline VecFloat VecFloat_Error() { + VecFloat v = { .len = -1 }; + return v; +} + inline VecT VecT_Error() { VecT v = { .len = -1 }; return v; @@ -222,15 +275,19 @@ inline VecNested VecVec_Error() { // Type objects extern PyTypeObject VecI64BufType; +extern PyTypeObject VecFloatBufType; extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; extern PyTypeObject VecI64Type; +extern PyTypeObject VecFloatType; extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; -extern VecI64Features I64Features; extern PyTypeObject *I64TypeObj; + +extern VecI64Features I64Features; +extern VecFloatFeatures FloatFeatures; extern VecTFeatures TFeatures; extern VecNestedFeatures TExtFeatures; @@ -245,6 +302,17 @@ VecI64 VecI64_Append(VecI64, int64_t x); VecI64 VecI64_Remove(VecI64, int64_t x); VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); +// vec[float] operations + +static inline int VecFloat_Check(PyObject *o) { + return o->ob_type == &VecFloatType; +} + +PyObject *VecFloat_Box(VecFloat); +VecFloat VecFloat_Append(VecFloat, double x); +VecFloat VecFloat_Remove(VecFloat, double x); +VecFloatPopResult VecFloat_Pop(VecFloat v, Py_ssize_t index); + // vec[t] operations static inline int VecT_Check(PyObject *o) { @@ -288,7 +356,6 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index); static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { - // TODO: vec[i64] if (item->ob_type == &VecTType) { VecNestedObject *o = (VecNestedObject *)item; if (o->vec.buf->item_type == v.buf->item_type) { @@ -301,6 +368,11 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; return 0; + } else if (item->ob_type == &VecFloatType && v.buf->item_type == VEC_ITEM_TYPE_FLOAT) { + VecFloatObject *o = (VecFloatObject *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + return 0; } } else if (item->ob_type == &VecNestedType) { VecNestedObject *o = (VecNestedObject *)item; @@ -331,6 +403,10 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { // vec[i64] VecI64 v = { .len = item.len, .buf = (VecI64BufObject *)item.buf }; return VecI64_Box(v); + } else if (item_type == VEC_ITEM_TYPE_FLOAT) { + // vec[float] + VecFloat v = { .len = item.len, .buf = (VecFloatBufObject *)item.buf }; + return VecFloat_Box(v); } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecTBufObject *)item.buf }; From 1abadd717c17310ceb8744a94933cb9771f6e87b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 16:21:11 +0000 Subject: [PATCH 136/208] Refactor --- mypyc/lib-rt/vecs/vec_template.c | 15 ++++++++++----- mypyc/lib-rt/vecs/vecs.h | 10 ---------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 31a0bfa32149c..23d8c974bf055 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -15,6 +15,11 @@ #include #include "vecs.h" +inline static VEC vec_error() { + VEC v = { .len = -1 }; + return v; +} + // Alloc a partially initialized vec. Caller *must* initialize len. static VEC vec_alloc(Py_ssize_t size) { @@ -25,7 +30,7 @@ static VEC vec_alloc(Py_ssize_t size) } else { buf = PyObject_NewVar(BUF_OBJECT, &BUF_TYPE, size); if (buf == NULL) - return FUNC(Error)(); + return vec_error(); } VEC res = { .buf = buf }; return res; @@ -52,7 +57,7 @@ VEC FUNC(Unbox)(PyObject *obj) { } else { // TODO: Better error message PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); - return FUNC(Error)(); + return vec_error(); } } @@ -262,7 +267,7 @@ VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { Py_ssize_t new_size = 2 * cap + 1; VEC new = vec_alloc(new_size); if (VEC_IS_ERROR(new)) - return FUNC(Error)(); + return vec_error(); new.len = vec.len + 1; if (vec.len > 0) memcpy(new.buf->items, vec.buf->items, sizeof(ITEM_C_TYPE) * vec.len); @@ -284,7 +289,7 @@ VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return FUNC(Error)(); + return vec_error(); } NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { @@ -295,7 +300,7 @@ NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = FUNC(Error)(); + result.f0 = vec_error(); result.f1 = 0; return result; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 187ac1fc60434..2e463e480d528 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -252,16 +252,6 @@ typedef struct { #define VEC_DECREF(v) Py_XDECREF((v).buf) #define VEC_INCREF(v) Py_XINCREF((v).buf) -inline VecI64 VecI64_Error() { - VecI64 v = { .len = -1 }; - return v; -} - -inline VecFloat VecFloat_Error() { - VecFloat v = { .len = -1 }; - return v; -} - inline VecT VecT_Error() { VecT v = { .len = -1 }; return v; From 772681ae57143916b22f98cab5bcbd8fe0f3c1f6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 16:22:42 +0000 Subject: [PATCH 137/208] Refactor --- mypyc/lib-rt/vecs/vec_nested.c | 20 ++++++++++++-------- mypyc/lib-rt/vecs/vec_t.c | 15 ++++++++++----- mypyc/lib-rt/vecs/vecs.h | 10 ---------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index da47491b1f4ca..7db16a228c4ce 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -9,17 +9,21 @@ #include #include "vecs.h" +static inline VecNested vec_error() { + VecNested v = { .len = -1 }; + return v; +} + static inline PyObject *box_vec_item_by_index(VecNested v, Py_ssize_t index) { return VecVec_BoxItem(v, v.buf->items[index]); } - // Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the // return value. static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { VecNestedBufObject *buf = PyObject_GC_NewVar(VecNestedBufObject, &VecNestedBufType, size); if (buf == NULL) - return VecVec_Error(); + return vec_error(); buf->item_type = item_type; buf->depth = depth; if (!Vec_IsMagicItemType(item_type)) @@ -48,7 +52,7 @@ VecNested VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { } // TODO: Better error message, with name of type PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return VecVec_Error(); + return vec_error(); } VecNested VecVec_ConvertFromNested(VecNestedBufItem item) { @@ -247,7 +251,7 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { PyObject *boxed_arg = VecVec_BoxItem(self, arg); if (boxed_arg == NULL) - return VecVec_Error(); + return vec_error(); for (Py_ssize_t i = 0; i < self.len; i++) { int match = 0; @@ -257,13 +261,13 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { PyObject *item = box_vec_item_by_index(self, i); if (item == NULL) { Py_DECREF(boxed_arg); - return VecVec_Error(); + return vec_error(); } int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); Py_DECREF(item); if (itemcmp < 0) { Py_DECREF(boxed_arg); - return VecVec_Error(); + return vec_error(); } match = itemcmp; } @@ -283,7 +287,7 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { } Py_DECREF(boxed_arg); PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return VecVec_Error(); + return vec_error(); } VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { @@ -294,7 +298,7 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = VecVec_Error(); + result.f0 = vec_error(); result.f1.len = 0; result.f1.buf = NULL; return result; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index b025f3974686e..1e34735306234 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -11,6 +11,11 @@ #include #include "vecs.h" +static inline VecT vec_error() { + VecT v = { .len = -1 }; + return v; +} + static inline VecTBufObject *alloc_buf(Py_ssize_t size, size_t item_type) { VecTBufObject *buf = PyObject_GC_NewVar(VecTBufObject, &VecTBufType, size); if (buf == NULL) @@ -31,7 +36,7 @@ static VecT vec_alloc(Py_ssize_t size, size_t item_type) { } else { buf = alloc_buf(size, item_type); if (buf == NULL) - return VecT_Error(); + return vec_error(); } return (VecT) { .buf = buf }; } @@ -62,7 +67,7 @@ VecT VecT_Unbox(PyObject *obj, size_t item_type) { } // TODO: Better error message, with name of type PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return VecT_Error(); + return vec_error(); } VecT VecT_ConvertFromNested(VecNestedBufItem item) { @@ -253,7 +258,7 @@ VecT VecT_Remove(VecT v, PyObject *arg) { else { int itemcmp = PyObject_RichCompareBool(items[i], arg, Py_EQ); if (itemcmp < 0) - return VecT_Error(); + return vec_error(); match = itemcmp; } if (match) { @@ -270,7 +275,7 @@ VecT VecT_Remove(VecT v, PyObject *arg) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - return VecT_Error(); + return vec_error(); } VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { @@ -281,7 +286,7 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); - result.f0 = VecT_Error(); + result.f0 = vec_error(); result.f1 = NULL; return result; } diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 2e463e480d528..93ced7ca103bd 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -252,16 +252,6 @@ typedef struct { #define VEC_DECREF(v) Py_XDECREF((v).buf) #define VEC_INCREF(v) Py_XINCREF((v).buf) -inline VecT VecT_Error() { - VecT v = { .len = -1 }; - return v; -} - -inline VecNested VecVec_Error() { - VecNested v = { .len = -1 }; - return v; -} - // Type objects extern PyTypeObject VecI64BufType; From 16539f14ad8420576e079f7d9fac365c68160950 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 18:07:25 +0000 Subject: [PATCH 138/208] Refactor --- mypyc/lib-rt/vecs/vec_float.c | 4 ++-- mypyc/lib-rt/vecs/vec_i64.c | 4 ++-- mypyc/lib-rt/vecs/vec_template.c | 9 ++------- mypyc/lib-rt/vecs/vecs.c | 20 ++++++++----------- mypyc/lib-rt/vecs/vecs.h | 34 ++++++++++++++++++++++++-------- 5 files changed, 40 insertions(+), 31 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c index 126bff8a790d5..da31e6070f02b 100644 --- a/mypyc/lib-rt/vecs/vec_float.c +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -11,7 +11,7 @@ #define FEATURES FloatFeatures #define BOX_ITEM(x) PyFloat_FromDouble(x) -#define UNBOX_ITEM(x) PyFloat_AsDouble(x) -#define IS_UNBOX_ERROR(x) ((x) == -1.0 && PyErr_Occurred()) +#define UNBOX_ITEM VecFloat_UnboxItem +#define IS_UNBOX_ERROR VecFloat_IsUnboxError #include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index cd7a0e20fa377..1dae325780ca3 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -11,7 +11,7 @@ #define FEATURES I64Features #define BOX_ITEM(x) PyLong_FromLongLong(x) -#define UNBOX_ITEM(x) PyLong_AsLongLong(x) -#define IS_UNBOX_ERROR(x) ((x) == -1 && PyErr_Occurred()) +#define UNBOX_ITEM VecI64_UnboxItem +#define IS_UNBOX_ERROR VecI64_IsUnboxError #include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 23d8c974bf055..c9c0397a2061d 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -202,19 +202,14 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { } static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - if (Vec_CheckFloatError(o)) + ITEM_C_TYPE x = UNBOX_ITEM(o); + if (IS_UNBOX_ERROR(x)) return -1; VEC v = ((VEC_OBJECT *)self)->vec; if ((size_t)i < (size_t)v.len) { - ITEM_C_TYPE x = UNBOX_ITEM(o); - if (IS_UNBOX_ERROR(x)) - return -1; v.buf->items[i] = x; return 0; } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - ITEM_C_TYPE x = UNBOX_ITEM(o); - if (IS_UNBOX_ERROR(x)) - return -1; v.buf->items[i + v.len] = x; return 0; } else { diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index ef2c113572c3c..bdb20385e8ade 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -351,10 +351,8 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return NULL; if (VecI64_Check(vec)) { - if (Vec_CheckFloatError(item)) - return NULL; - int64_t x = PyLong_AsLong(item); - if (x == -1 && PyErr_Occurred()) { + int64_t x = VecI64_UnboxItem(item); + if (VecI64_IsUnboxError(x)) { return NULL; } VecI64 v = ((VecI64Object *)vec)->vec; @@ -364,8 +362,8 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return NULL; // TODO: decref? return VecI64_Box(v); } else if (VecFloat_Check(vec)) { - double x = PyFloat_AsDouble(item); - if (x == -1 && PyErr_Occurred()) { + double x = VecFloat_UnboxItem(item); + if (VecFloat_IsUnboxError(x)) { return NULL; } VecFloat v = ((VecFloatObject *)vec)->vec; @@ -409,10 +407,8 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return NULL; if (VecI64_Check(vec)) { - if (Vec_CheckFloatError(item)) - return NULL; - int64_t x = PyLong_AsLong(item); - if (x == -1 && PyErr_Occurred()) { + int64_t x = VecI64_UnboxItem(item); + if (VecI64_IsUnboxError(x)) { return NULL; } VecI64 v = ((VecI64Object *)vec)->vec; @@ -422,8 +418,8 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return NULL; // TODO: decref? return VecI64_Box(v); } else if (VecFloat_Check(vec)) { - double x = PyFloat_AsDouble(item); - if (x == -1 && PyErr_Occurred()) { + double x = VecFloat_UnboxItem(item); + if (VecFloat_IsUnboxError(x)) { return NULL; } VecFloat v = ((VecFloatObject *)vec)->vec; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 93ced7ca103bd..7733147cd5b80 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -273,10 +273,28 @@ extern VecNestedFeatures TExtFeatures; // vec[i64] operations +static inline int Vec_CheckFloatError(PyObject *o) { + if (PyFloat_Check(o)) { + PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); + return 1; + } + return 0; +} + static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } +static inline int64_t VecI64_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + return PyLong_AsLongLong(o); +} + +static inline int VecI64_IsUnboxError(int64_t x) { + return x == -1 && PyErr_Occurred(); +} + PyObject *VecI64_Box(VecI64); VecI64 VecI64_Append(VecI64, int64_t x); VecI64 VecI64_Remove(VecI64, int64_t x); @@ -288,6 +306,14 @@ static inline int VecFloat_Check(PyObject *o) { return o->ob_type == &VecFloatType; } +static inline double VecFloat_UnboxItem(PyObject *o) { + return PyFloat_AsDouble(o); +} + +static inline int VecFloat_IsUnboxError(double x) { + return x == -1.0 && PyErr_Occurred(); +} + PyObject *VecFloat_Box(VecFloat); VecFloat VecFloat_Append(VecFloat, double x); VecFloat VecFloat_Remove(VecFloat, double x); @@ -397,14 +423,6 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { // Misc helpers -static inline int Vec_CheckFloatError(PyObject *o) { - if (PyFloat_Check(o)) { - PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); - return 1; - } - return 0; -} - PyObject *Vec_TypeToStr(size_t item_type, size_t depth); PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose); PyObject *Vec_GenericRichcompare(Py_ssize_t *len, PyObject **items, From b0385c44ec68fe4a6192173b9bc71fcc6290c91b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 18:13:50 +0000 Subject: [PATCH 139/208] Refactor --- mypyc/lib-rt/vecs/vec_float.c | 2 +- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vecs.c | 15 ++++----------- mypyc/lib-rt/vecs/vecs.h | 8 ++++++++ 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c index da31e6070f02b..90cce730029e8 100644 --- a/mypyc/lib-rt/vecs/vec_float.c +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -10,7 +10,7 @@ #define ITEM_C_TYPE double #define FEATURES FloatFeatures -#define BOX_ITEM(x) PyFloat_FromDouble(x) +#define BOX_ITEM VecFloat_BoxItem #define UNBOX_ITEM VecFloat_UnboxItem #define IS_UNBOX_ERROR VecFloat_IsUnboxError diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index 1dae325780ca3..dae31c8048938 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -10,7 +10,7 @@ #define ITEM_C_TYPE int64_t #define FEATURES I64Features -#define BOX_ITEM(x) PyLong_FromLongLong(x) +#define BOX_ITEM VecI64_BoxItem #define UNBOX_ITEM VecI64_UnboxItem #define IS_UNBOX_ERROR VecI64_IsUnboxError diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index bdb20385e8ade..631d297466194 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -471,29 +471,24 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) r = VecI64_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - - result_item0 = VecI64_Box(r.f0); - if (result_item0 == NULL) + if ((result_item0 = VecI64_Box(r.f0)) == NULL) return NULL; - result_item1 = PyLong_FromLongLong(r.f1); + result_item1 = VecI64_BoxItem(r.f1); } else if (VecFloat_Check(vec)) { VecFloat v = ((VecFloatObject *)vec)->vec; VecFloatPopResult r; r = VecFloat_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - - result_item0 = VecFloat_Box(r.f0); - if (result_item0 == NULL) + if ((result_item0 = VecFloat_Box(r.f0)) == NULL) return NULL; - result_item1 = PyFloat_FromDouble(r.f1); + result_item1 = VecFloat_BoxItem(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; r = VecT_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = VecT_Box(r.f0, v.buf->item_type); if (result_item0 == NULL) { Py_DECREF(r.f1); @@ -506,14 +501,12 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) r = VecVec_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = VecVec_Box(r.f0); if (result_item0 == NULL) { Py_DECREF(r.f0.buf); Py_DECREF(r.f1.buf); return NULL; } - result_item1 = VecVec_BoxItem(r.f0, r.f1); if (result_item1 == NULL) { Py_DECREF(result_item0); diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 7733147cd5b80..6caca7bd151fa 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -285,6 +285,10 @@ static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } +static inline PyObject *VecI64_BoxItem(int64_t x) { + return PyLong_FromLongLong(x); +} + static inline int64_t VecI64_UnboxItem(PyObject *o) { if (Vec_CheckFloatError(o)) return -1; @@ -306,6 +310,10 @@ static inline int VecFloat_Check(PyObject *o) { return o->ob_type == &VecFloatType; } +static inline PyObject *VecFloat_BoxItem(double x) { + return PyFloat_FromDouble(x); +} + static inline double VecFloat_UnboxItem(PyObject *o) { return PyFloat_AsDouble(o); } From f42fb5bc23560e9e236511453f214c403a1eb9a4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 1 Jan 2024 18:48:27 +0000 Subject: [PATCH 140/208] Implement vec[u8] --- mypyc/lib-rt/vecs/vec_u8.c | 17 +++++++ mypyc/lib-rt/vecs/vecs.c | 51 ++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 95 +++++++++++++++++++++++++++++++++++++- 3 files changed, 161 insertions(+), 2 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_u8.c diff --git a/mypyc/lib-rt/vecs/vec_u8.c b/mypyc/lib-rt/vecs/vec_u8.c new file mode 100644 index 0000000000000..94976c16cde92 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_u8.c @@ -0,0 +1,17 @@ +#define VEC VecU8 +#define VEC_TYPE VecU8Type +#define VEC_OBJECT VecU8Object +#define BUF_OBJECT VecU8BufObject +#define BUF_TYPE VecU8BufType +#define NAME(suffix) VecU8##suffix +#define FUNC(suffix) VecU8_##suffix +#define ITEM_TYPE_STR "u8" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_U8 +#define ITEM_C_TYPE uint8_t +#define FEATURES U8Features + +#define BOX_ITEM VecU8_BoxItem +#define UNBOX_ITEM VecU8_UnboxItem +#define IS_UNBOX_ERROR VecU8_IsUnboxError + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 631d297466194..fd583a968e625 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -4,6 +4,7 @@ PyTypeObject *I64TypeObj; +PyTypeObject *U8TypeObj; // vec type proxy @@ -70,6 +71,8 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { if (depth == 0) if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { item = PyUnicode_FromFormat("i64"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_U8) { + item = PyUnicode_FromFormat("u8"); } else if ((item_type & ~1) == VEC_ITEM_TYPE_FLOAT) { item = PyUnicode_FromFormat("float"); } else { @@ -147,6 +150,9 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (item == (PyObject *)I64TypeObj) { Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; + } else if (item == (PyObject *)U8TypeObj) { + Py_INCREF(&VecU8Type); + return (PyObject *)&VecU8Type; } else if (item == (PyObject *)&PyFloat_Type) { Py_INCREF(&VecFloatType); return (PyObject *)&VecFloatType; @@ -179,6 +185,10 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) item_type = VEC_ITEM_TYPE_I64; depth = 1; // TODO: Check optionals? + } else if (item == (PyObject *)&VecU8Type) { + item_type = VEC_ITEM_TYPE_U8; + depth = 1; + // TODO: Check optionals? } else if (item == (PyObject *)&VecFloatType) { item_type = VEC_ITEM_TYPE_FLOAT; depth = 1; @@ -361,6 +371,17 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI64_Box(v); + } else if (VecU8_Check(vec)) { + uint8_t x = VecU8_UnboxItem(item); + if (VecU8_IsUnboxError(x)) { + return NULL; + } + VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); + v = VecU8_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecU8_Box(v); } else if (VecFloat_Check(vec)) { double x = VecFloat_UnboxItem(item); if (VecFloat_IsUnboxError(x)) { @@ -417,6 +438,17 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI64_Box(v); + } else if (VecU8_Check(vec)) { + uint8_t x = VecU8_UnboxItem(item); + if (VecU8_IsUnboxError(x)) { + return NULL; + } + VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); + v = VecU8_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecU8_Box(v); } else if (VecFloat_Check(vec)) { double x = VecFloat_UnboxItem(item); if (VecFloat_IsUnboxError(x)) { @@ -474,6 +506,15 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if ((result_item0 = VecI64_Box(r.f0)) == NULL) return NULL; result_item1 = VecI64_BoxItem(r.f1); + } else if (VecU8_Check(vec)) { + VecU8 v = ((VecU8Object *)vec)->vec; + VecU8PopResult r; + r = VecU8_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecU8_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecU8_BoxItem(r.f1); } else if (VecFloat_Check(vec)) { VecFloat v = ((VecFloatObject *)vec)->vec; VecFloatPopResult r; @@ -554,6 +595,7 @@ static VecCapsule Capsule = { &TFeatures, &TExtFeatures, &I64Features, + &U8Features, &FloatFeatures, }; @@ -570,6 +612,11 @@ PyInit_vecs(void) if (I64TypeObj == NULL) { return NULL; } + U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); + // TODO: Check that it's a type object! + if (U8TypeObj == NULL) { + return NULL; + } if (PyType_Ready(&VecGenericType) < 0) return NULL; @@ -590,6 +637,10 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecI64BufType) < 0) return NULL; + if (PyType_Ready(&VecU8Type) < 0) + return NULL; + if (PyType_Ready(&VecU8BufType) < 0) + return NULL; if (PyType_Ready(&VecFloatType) < 0) return NULL; if (PyType_Ready(&VecFloatBufType) < 0) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 6caca7bd151fa..ad72366ed73b1 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -10,6 +10,7 @@ // Item type constants; must be even but not multiples of 4 (2 + 4 * n) #define VEC_ITEM_TYPE_I64 2 +#define VEC_ITEM_TYPE_U8 14 #define VEC_ITEM_TYPE_FLOAT 18 inline size_t Vec_IsMagicItemType(size_t item_type) { @@ -26,6 +27,12 @@ typedef struct _VecI64BufObject { int64_t items[1]; } VecI64BufObject; +// vecbuf[u8] +typedef struct _VecU8BufObject { + PyObject_VAR_HEAD + uint8_t items[1]; +} VecU8BufObject; + // vecbuf[float] typedef struct _VecFloatBufObject { PyObject_VAR_HEAD @@ -65,6 +72,11 @@ typedef struct _VecI64 { VecI64BufObject *buf; } VecI64; +typedef struct _VecU8 { + Py_ssize_t len; + VecU8BufObject *buf; +} VecU8; + typedef struct _VecFloat { Py_ssize_t len; VecFloatBufObject *buf; @@ -96,6 +108,12 @@ typedef struct _VecI64Object { VecI64 vec; } VecI64Object; +// Boxed vec[u8] +typedef struct _VecU8Object { + PyObject_HEAD + VecU8 vec; +} VecU8Object; + // Boxed vec[float] typedef struct _VecFloatObject { PyObject_HEAD @@ -124,6 +142,15 @@ typedef struct tuple_T2V88 { static tuple_T2V88 tuple_undefined_T2V88 = { { -1, NULL } , 0 }; #endif +#ifndef MYPYC_DECLARED_tuple_T2VU1U1 +#define MYPYC_DECLARED_tuple_T2VU1U1 +typedef struct tuple_T2VU1U1 { + VecU8 f0; + uint8_t f1; +} tuple_T2VU1U1; +static tuple_T2VU1U1 tuple_undefined_T2VU1U1 = { { -1, NULL } , 0 }; +#endif + #ifndef MYPYC_DECLARED_tuple_T2VFF #define MYPYC_DECLARED_tuple_T2VFF typedef struct tuple_T2VFF { @@ -134,6 +161,7 @@ static tuple_T2VFF tuple_undefined_T2VFF = { { -1, NULL } , 0.0 }; #endif typedef tuple_T2V88 VecI64PopResult; +typedef tuple_T2VU1U1 VecU8PopResult; typedef tuple_T2VFF VecFloatPopResult; // vec[i64] operations + type objects (stored in a capsule) @@ -155,6 +183,25 @@ typedef struct _VecI64Features { // iter? } VecI64Features; +// vec[u8] operations + type objects (stored in a capsule) +typedef struct _VecU8Features { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecU8 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecU8); + VecU8 (*unbox)(PyObject *); + VecU8 (*convert_from_nested)(VecNestedBufItem); + VecU8 (*append)(VecU8, uint8_t); + VecU8PopResult (*pop)(VecU8, Py_ssize_t); + VecU8 (*remove)(VecU8, uint8_t); + // TODO: Py_ssize_t + VecU8 (*slice)(VecU8, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, uint8_t); + // iter? +} VecU8Features; + // vec[float] operations + type objects (stored in a capsule) typedef struct _VecFloatFeatures { PyTypeObject *boxed_type; @@ -241,6 +288,7 @@ typedef struct { VecTFeatures *t; VecNestedFeatures *nested; VecI64Features *i64; + VecU8Features *u8; VecFloatFeatures *float_; } VecCapsule; @@ -255,18 +303,22 @@ typedef struct { // Type objects extern PyTypeObject VecI64BufType; +extern PyTypeObject VecU8BufType; extern PyTypeObject VecFloatBufType; extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; extern PyTypeObject VecI64Type; +extern PyTypeObject VecU8Type; extern PyTypeObject VecFloatType; extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; extern PyTypeObject *I64TypeObj; +extern PyTypeObject *U8TypeObj; extern VecI64Features I64Features; +extern VecU8Features U8Features; extern VecFloatFeatures FloatFeatures; extern VecTFeatures TFeatures; extern VecNestedFeatures TExtFeatures; @@ -304,6 +356,39 @@ VecI64 VecI64_Append(VecI64, int64_t x); VecI64 VecI64_Remove(VecI64, int64_t x); VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); +// vec[u8] operations + +static inline int VecU8_Check(PyObject *o) { + return o->ob_type == &VecU8Type; +} + +static inline PyObject *VecU8_BoxItem(uint8_t x) { + return PyLong_FromUnsignedLong(x); +} + +static inline uint8_t VecU8_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + unsigned long x = PyLong_AsUnsignedLong(o); + if (x <= 255) + return x; + else if (x == (unsigned long)-1) + return 239; + else { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to u8"); + return 239; + } +} + +static inline int VecU8_IsUnboxError(uint8_t x) { + return x == 239 && PyErr_Occurred(); +} + +PyObject *VecU8_Box(VecU8); +VecU8 VecU8_Append(VecU8, uint8_t x); +VecU8 VecU8_Remove(VecU8, uint8_t x); +VecU8PopResult VecU8_Pop(VecU8 v, Py_ssize_t index); + // vec[float] operations static inline int VecFloat_Check(PyObject *o) { @@ -382,6 +467,11 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; return 0; + } else if (item->ob_type == &VecU8Type && v.buf->item_type == VEC_ITEM_TYPE_U8) { + VecU8Object *o = (VecU8Object *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + return 0; } else if (item->ob_type == &VecFloatType && v.buf->item_type == VEC_ITEM_TYPE_FLOAT) { VecFloatObject *o = (VecFloatObject *)item; unboxed->len = o->vec.len; @@ -414,11 +504,12 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { // Item is a non-nested vec size_t item_type = v.buf->item_type; if (item_type == VEC_ITEM_TYPE_I64) { - // vec[i64] VecI64 v = { .len = item.len, .buf = (VecI64BufObject *)item.buf }; return VecI64_Box(v); + } else if (item_type == VEC_ITEM_TYPE_U8) { + VecU8 v = { .len = item.len, .buf = (VecU8BufObject *)item.buf }; + return VecU8_Box(v); } else if (item_type == VEC_ITEM_TYPE_FLOAT) { - // vec[float] VecFloat v = { .len = item.len, .buf = (VecFloatBufObject *)item.buf }; return VecFloat_Box(v); } else { From adb5b07aca2ffcbf010af2369433e09842cfdea4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Jan 2024 12:17:02 +0000 Subject: [PATCH 141/208] Support vec[i32] and vec[i16] --- mypyc/lib-rt/vecs/vec_i16.c | 17 ++++ mypyc/lib-rt/vecs/vec_i32.c | 17 ++++ mypyc/lib-rt/vecs/vecs.c | 102 ++++++++++++++++++++ mypyc/lib-rt/vecs/vecs.h | 182 +++++++++++++++++++++++++++++++++++- 4 files changed, 314 insertions(+), 4 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_i16.c create mode 100644 mypyc/lib-rt/vecs/vec_i32.c diff --git a/mypyc/lib-rt/vecs/vec_i16.c b/mypyc/lib-rt/vecs/vec_i16.c new file mode 100644 index 0000000000000..d4b0dd0e92cd9 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i16.c @@ -0,0 +1,17 @@ +#define VEC VecI16 +#define VEC_TYPE VecI16Type +#define VEC_OBJECT VecI16Object +#define BUF_OBJECT VecI16BufObject +#define BUF_TYPE VecI16BufType +#define NAME(suffix) VecI16##suffix +#define FUNC(suffix) VecI16_##suffix +#define ITEM_TYPE_STR "i16" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I16 +#define ITEM_C_TYPE int16_t +#define FEATURES I16Features + +#define BOX_ITEM VecI16_BoxItem +#define UNBOX_ITEM VecI16_UnboxItem +#define IS_UNBOX_ERROR VecI16_IsUnboxError + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vec_i32.c b/mypyc/lib-rt/vecs/vec_i32.c new file mode 100644 index 0000000000000..a51b485a61f28 --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_i32.c @@ -0,0 +1,17 @@ +#define VEC VecI32 +#define VEC_TYPE VecI32Type +#define VEC_OBJECT VecI32Object +#define BUF_OBJECT VecI32BufObject +#define BUF_TYPE VecI32BufType +#define NAME(suffix) VecI32##suffix +#define FUNC(suffix) VecI32_##suffix +#define ITEM_TYPE_STR "i32" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I32 +#define ITEM_C_TYPE int32_t +#define FEATURES I32Features + +#define BOX_ITEM VecI32_BoxItem +#define UNBOX_ITEM VecI32_UnboxItem +#define IS_UNBOX_ERROR VecI32_IsUnboxError + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index fd583a968e625..6c72688f89ad5 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -4,6 +4,8 @@ PyTypeObject *I64TypeObj; +PyTypeObject *I32TypeObj; +PyTypeObject *I16TypeObj; PyTypeObject *U8TypeObj; @@ -75,6 +77,10 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { item = PyUnicode_FromFormat("u8"); } else if ((item_type & ~1) == VEC_ITEM_TYPE_FLOAT) { item = PyUnicode_FromFormat("float"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_I32) { + item = PyUnicode_FromFormat("i32"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_I16) { + item = PyUnicode_FromFormat("i16"); } else { item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); if (item_type & 1) { @@ -156,6 +162,12 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } else if (item == (PyObject *)&PyFloat_Type) { Py_INCREF(&VecFloatType); return (PyObject *)&VecFloatType; + } else if (item == (PyObject *)I32TypeObj) { + Py_INCREF(&VecI32Type); + return (PyObject *)&VecI32Type; + } else if (item == (PyObject *)I16TypeObj) { + Py_INCREF(&VecI16Type); + return (PyObject *)&VecI16Type; } else { size_t depth = 0; size_t item_type; @@ -193,6 +205,14 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) item_type = VEC_ITEM_TYPE_FLOAT; depth = 1; // TODO: Check optionals? + } else if (item == (PyObject *)&VecI32Type) { + item_type = VEC_ITEM_TYPE_I32; + depth = 1; + // TODO: Check optionals? + } else if (item == (PyObject *)&VecI16Type) { + item_type = VEC_ITEM_TYPE_I16; + depth = 1; + // TODO: Check optionals? } else { item_type = (size_t)item; } @@ -393,6 +413,28 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecFloat_Box(v); + } else if (VecI32_Check(vec)) { + int32_t x = VecI32_UnboxItem(item); + if (VecI32_IsUnboxError(x)) { + return NULL; + } + VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); + v = VecI32_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecI32_Box(v); + } else if (VecI16_Check(vec)) { + int16_t x = VecI16_UnboxItem(item); + if (VecI16_IsUnboxError(x)) { + return NULL; + } + VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); + v = VecI16_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecI16_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -460,6 +502,28 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecFloat_Box(v); + } else if (VecI32_Check(vec)) { + int32_t x = VecI32_UnboxItem(item); + if (VecI32_IsUnboxError(x)) { + return NULL; + } + VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); + v = VecI32_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecI32_Box(v); + } else if (VecI16_Check(vec)) { + int16_t x = VecI16_UnboxItem(item); + if (VecI16_IsUnboxError(x)) { + return NULL; + } + VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); + v = VecI16_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecI16_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -524,6 +588,24 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if ((result_item0 = VecFloat_Box(r.f0)) == NULL) return NULL; result_item1 = VecFloat_BoxItem(r.f1); + } else if (VecI32_Check(vec)) { + VecI32 v = ((VecI32Object *)vec)->vec; + VecI32PopResult r; + r = VecI32_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecI32_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecI32_BoxItem(r.f1); + } else if (VecI16_Check(vec)) { + VecI16 v = ((VecI16Object *)vec)->vec; + VecI16PopResult r; + r = VecI16_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecI16_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecI16_BoxItem(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; @@ -595,6 +677,8 @@ static VecCapsule Capsule = { &TFeatures, &TExtFeatures, &I64Features, + &I32Features, + &I16Features, &U8Features, &FloatFeatures, }; @@ -612,6 +696,16 @@ PyInit_vecs(void) if (I64TypeObj == NULL) { return NULL; } + I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); + // TODO: Check that it's a type object! + if (I32TypeObj == NULL) { + return NULL; + } + I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); + // TODO: Check that it's a type object! + if (I16TypeObj == NULL) { + return NULL; + } U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); // TODO: Check that it's a type object! if (U8TypeObj == NULL) { @@ -637,6 +731,14 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecI64BufType) < 0) return NULL; + if (PyType_Ready(&VecI32Type) < 0) + return NULL; + if (PyType_Ready(&VecI32BufType) < 0) + return NULL; + if (PyType_Ready(&VecI16Type) < 0) + return NULL; + if (PyType_Ready(&VecI16BufType) < 0) + return NULL; if (PyType_Ready(&VecU8Type) < 0) return NULL; if (PyType_Ready(&VecU8BufType) < 0) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index ad72366ed73b1..834db24594312 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -8,9 +8,15 @@ // use PyErr_Occurred() since this overlaps with valid integer values. #define MYPYC_INT_ERROR -113 -// Item type constants; must be even but not multiples of 4 (2 + 4 * n) +// Item type constants for supported packed/specialized item types; must be +// even but not a multiple of 4 (2 + 4 * n). Each of these has a corresponding +// distinct implementation C extension class. For example, vec[i64] has a +// different runtime type than vec[i32]. All other item types use generic +// implementations. #define VEC_ITEM_TYPE_I64 2 -#define VEC_ITEM_TYPE_U8 14 +#define VEC_ITEM_TYPE_I32 6 +#define VEC_ITEM_TYPE_I16 10 +#define VEC_ITEM_TYPE_U8 14 #define VEC_ITEM_TYPE_FLOAT 18 inline size_t Vec_IsMagicItemType(size_t item_type) { @@ -27,6 +33,18 @@ typedef struct _VecI64BufObject { int64_t items[1]; } VecI64BufObject; +// vecbuf[i32] +typedef struct _VecI32BufObject { + PyObject_VAR_HEAD + int32_t items[1]; +} VecI32BufObject; + +// vecbuf[i16] +typedef struct _VecI16BufObject { + PyObject_VAR_HEAD + int16_t items[1]; +} VecI16BufObject; + // vecbuf[u8] typedef struct _VecU8BufObject { PyObject_VAR_HEAD @@ -72,6 +90,16 @@ typedef struct _VecI64 { VecI64BufObject *buf; } VecI64; +typedef struct _VecI32 { + Py_ssize_t len; + VecI32BufObject *buf; +} VecI32; + +typedef struct _VecI16 { + Py_ssize_t len; + VecI16BufObject *buf; +} VecI16; + typedef struct _VecU8 { Py_ssize_t len; VecU8BufObject *buf; @@ -108,6 +136,18 @@ typedef struct _VecI64Object { VecI64 vec; } VecI64Object; +// Boxed vec[i32] +typedef struct _VecI32Object { + PyObject_HEAD + VecI32 vec; +} VecI32Object; + +// Boxed vec[i16] +typedef struct _VecI16Object { + PyObject_HEAD + VecI16 vec; +} VecI16Object; + // Boxed vec[u8] typedef struct _VecU8Object { PyObject_HEAD @@ -142,6 +182,24 @@ typedef struct tuple_T2V88 { static tuple_T2V88 tuple_undefined_T2V88 = { { -1, NULL } , 0 }; #endif +#ifndef MYPYC_DECLARED_tuple_T2V44 +#define MYPYC_DECLARED_tuple_T2V44 +typedef struct tuple_T2V44 { + VecI32 f0; + int32_t f1; +} tuple_T2V44; +static tuple_T2V44 tuple_undefined_T2V44 = { { -1, NULL } , 0 }; +#endif + +#ifndef MYPYC_DECLARED_tuple_T2V22 +#define MYPYC_DECLARED_tuple_T2V22 +typedef struct tuple_T2V22 { + VecI16 f0; + int16_t f1; +} tuple_T2V22; +static tuple_T2V22 tuple_undefined_T2V22 = { { -1, NULL } , 0 }; +#endif + #ifndef MYPYC_DECLARED_tuple_T2VU1U1 #define MYPYC_DECLARED_tuple_T2VU1U1 typedef struct tuple_T2VU1U1 { @@ -161,6 +219,8 @@ static tuple_T2VFF tuple_undefined_T2VFF = { { -1, NULL } , 0.0 }; #endif typedef tuple_T2V88 VecI64PopResult; +typedef tuple_T2V44 VecI32PopResult; +typedef tuple_T2V22 VecI16PopResult; typedef tuple_T2VU1U1 VecU8PopResult; typedef tuple_T2VFF VecFloatPopResult; @@ -183,6 +243,44 @@ typedef struct _VecI64Features { // iter? } VecI64Features; +// vec[i32] operations + type objects (stored in a capsule) +typedef struct _VecI32Features { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI32 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecI32); + VecI32 (*unbox)(PyObject *); + VecI32 (*convert_from_nested)(VecNestedBufItem); + VecI32 (*append)(VecI32, int32_t); + VecI32PopResult (*pop)(VecI32, Py_ssize_t); + VecI32 (*remove)(VecI32, int32_t); + // TODO: Py_ssize_t + VecI32 (*slice)(VecI32, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, int32_t); + // iter? +} VecI32Features; + +// vec[i16] operations + type objects (stored in a capsule) +typedef struct _VecI16Features { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecI16 (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecI16); + VecI16 (*unbox)(PyObject *); + VecI16 (*convert_from_nested)(VecNestedBufItem); + VecI16 (*append)(VecI16, int16_t); + VecI16PopResult (*pop)(VecI16, Py_ssize_t); + VecI16 (*remove)(VecI16, int16_t); + // TODO: Py_ssize_t + VecI16 (*slice)(VecI16, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, int16_t); + // iter? +} VecI16Features; + // vec[u8] operations + type objects (stored in a capsule) typedef struct _VecU8Features { PyTypeObject *boxed_type; @@ -288,6 +386,8 @@ typedef struct { VecTFeatures *t; VecNestedFeatures *nested; VecI64Features *i64; + VecI32Features *i32; + VecI16Features *i16; VecU8Features *u8; VecFloatFeatures *float_; } VecCapsule; @@ -303,28 +403,34 @@ typedef struct { // Type objects extern PyTypeObject VecI64BufType; +extern PyTypeObject VecI32BufType; +extern PyTypeObject VecI16BufType; extern PyTypeObject VecU8BufType; extern PyTypeObject VecFloatBufType; extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; extern PyTypeObject VecI64Type; +extern PyTypeObject VecI32Type; +extern PyTypeObject VecI16Type; extern PyTypeObject VecU8Type; extern PyTypeObject VecFloatType; extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; extern PyTypeObject *I64TypeObj; +extern PyTypeObject *I32TypeObj; +extern PyTypeObject *I16TypeObj; extern PyTypeObject *U8TypeObj; extern VecI64Features I64Features; +extern VecI32Features I32Features; +extern VecI16Features I16Features; extern VecU8Features U8Features; extern VecFloatFeatures FloatFeatures; extern VecTFeatures TFeatures; extern VecNestedFeatures TExtFeatures; -// vec[i64] operations - static inline int Vec_CheckFloatError(PyObject *o) { if (PyFloat_Check(o)) { PyErr_SetString(PyExc_TypeError, "integer argument expected, got float"); @@ -333,6 +439,8 @@ static inline int Vec_CheckFloatError(PyObject *o) { return 0; } +// vec[i64] operations + static inline int VecI64_Check(PyObject *o) { return o->ob_type == &VecI64Type; } @@ -356,6 +464,56 @@ VecI64 VecI64_Append(VecI64, int64_t x); VecI64 VecI64_Remove(VecI64, int64_t x); VecI64PopResult VecI64_Pop(VecI64 v, Py_ssize_t index); +// vec[i32] operations + +static inline int VecI32_Check(PyObject *o) { + return o->ob_type == &VecI32Type; +} + +static inline PyObject *VecI32_BoxItem(int32_t x) { + return PyLong_FromLongLong(x); +} + +static inline int32_t VecI32_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + return PyLong_AsLongLong(o); +} + +static inline int VecI32_IsUnboxError(int32_t x) { + return x == -1 && PyErr_Occurred(); +} + +PyObject *VecI32_Box(VecI32); +VecI32 VecI32_Append(VecI32, int32_t x); +VecI32 VecI32_Remove(VecI32, int32_t x); +VecI32PopResult VecI32_Pop(VecI32 v, Py_ssize_t index); + +// vec[i16] operations + +static inline int VecI16_Check(PyObject *o) { + return o->ob_type == &VecI16Type; +} + +static inline PyObject *VecI16_BoxItem(int16_t x) { + return PyLong_FromLongLong(x); +} + +static inline int16_t VecI16_UnboxItem(PyObject *o) { + if (Vec_CheckFloatError(o)) + return -1; + return PyLong_AsLongLong(o); +} + +static inline int VecI16_IsUnboxError(int16_t x) { + return x == -1 && PyErr_Occurred(); +} + +PyObject *VecI16_Box(VecI16); +VecI16 VecI16_Append(VecI16, int16_t x); +VecI16 VecI16_Remove(VecI16, int16_t x); +VecI16PopResult VecI16_Pop(VecI16 v, Py_ssize_t index); + // vec[u8] operations static inline int VecU8_Check(PyObject *o) { @@ -477,6 +635,16 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; return 0; + } else if (item->ob_type == &VecI32Type && v.buf->item_type == VEC_ITEM_TYPE_I32) { + VecI32Object *o = (VecI32Object *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + return 0; + } else if (item->ob_type == &VecI16Type && v.buf->item_type == VEC_ITEM_TYPE_I16) { + VecI16Object *o = (VecI16Object *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + return 0; } } else if (item->ob_type == &VecNestedType) { VecNestedObject *o = (VecNestedObject *)item; @@ -512,6 +680,12 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { } else if (item_type == VEC_ITEM_TYPE_FLOAT) { VecFloat v = { .len = item.len, .buf = (VecFloatBufObject *)item.buf }; return VecFloat_Box(v); + } else if (item_type == VEC_ITEM_TYPE_I32) { + VecI32 v = { .len = item.len, .buf = (VecI32BufObject *)item.buf }; + return VecI32_Box(v); + } else if (item_type == VEC_ITEM_TYPE_I16) { + VecI16 v = { .len = item.len, .buf = (VecI16BufObject *)item.buf }; + return VecI16_Box(v); } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecTBufObject *)item.buf }; From 7c5aa3ccf0088f48dc8ace1d677d69c34457e7dc Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Jan 2024 16:32:15 +0000 Subject: [PATCH 142/208] Add item overflow checks --- mypyc/lib-rt/vecs/vecs.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 834db24594312..3e9b30a27675f 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -477,7 +477,12 @@ static inline PyObject *VecI32_BoxItem(int32_t x) { static inline int32_t VecI32_UnboxItem(PyObject *o) { if (Vec_CheckFloatError(o)) return -1; - return PyLong_AsLongLong(o); + long x = PyLong_AsLong(o); + if (x >= 0x80000000L || x < -0x80000000L) { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to i32"); + return -1; + } + return x; } static inline int VecI32_IsUnboxError(int32_t x) { @@ -502,7 +507,12 @@ static inline PyObject *VecI16_BoxItem(int16_t x) { static inline int16_t VecI16_UnboxItem(PyObject *o) { if (Vec_CheckFloatError(o)) return -1; - return PyLong_AsLongLong(o); + long x = PyLong_AsLong(o); + if (x >= 32768 || x < -32768) { + PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to i16"); + return -1; + } + return x; } static inline int VecI16_IsUnboxError(int16_t x) { From 1dc7b4a84e309dbf5c1f57668b08b5f76ac2e3fa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Sat, 27 Jan 2024 17:32:46 +0000 Subject: [PATCH 143/208] Support vec[bool] --- mypyc/lib-rt/vecs/vec_bool.c | 17 +++++++ mypyc/lib-rt/vecs/vecs.c | 49 +++++++++++++++++-- mypyc/lib-rt/vecs/vecs.h | 95 ++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 3 deletions(-) create mode 100644 mypyc/lib-rt/vecs/vec_bool.c diff --git a/mypyc/lib-rt/vecs/vec_bool.c b/mypyc/lib-rt/vecs/vec_bool.c new file mode 100644 index 0000000000000..6217c8f58d48a --- /dev/null +++ b/mypyc/lib-rt/vecs/vec_bool.c @@ -0,0 +1,17 @@ +#define VEC VecBool +#define VEC_TYPE VecBoolType +#define VEC_OBJECT VecBoolObject +#define BUF_OBJECT VecBoolBufObject +#define BUF_TYPE VecBoolBufType +#define NAME(suffix) VecBool##suffix +#define FUNC(suffix) VecBool_##suffix +#define ITEM_TYPE_STR "bool" +#define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_BOOL +#define ITEM_C_TYPE char +#define FEATURES BoolFeatures + +#define BOX_ITEM VecBool_BoxItem +#define UNBOX_ITEM VecBool_UnboxItem +#define IS_UNBOX_ERROR VecBool_IsUnboxError + +#include "vec_template.c" diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/vecs.c index 6c72688f89ad5..b9a56a1daf7b3 100644 --- a/mypyc/lib-rt/vecs/vecs.c +++ b/mypyc/lib-rt/vecs/vecs.c @@ -81,6 +81,8 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { item = PyUnicode_FromFormat("i32"); } else if ((item_type & ~1) == VEC_ITEM_TYPE_I16) { item = PyUnicode_FromFormat("i16"); + } else if ((item_type & ~1) == VEC_ITEM_TYPE_BOOL) { + item = PyUnicode_FromFormat("bool"); } else { item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); if (item_type & 1) { @@ -168,6 +170,9 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } else if (item == (PyObject *)I16TypeObj) { Py_INCREF(&VecI16Type); return (PyObject *)&VecI16Type; + } else if (item == (PyObject *)&PyBool_Type) { + Py_INCREF(&VecBoolType); + return (PyObject *)&VecBoolType; } else { size_t depth = 0; size_t item_type; @@ -213,13 +218,15 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) item_type = VEC_ITEM_TYPE_I16; depth = 1; // TODO: Check optionals? + } else if (item == (PyObject *)&VecBoolType) { + item_type = VEC_ITEM_TYPE_BOOL; + depth = 1; + // TODO: Check optionals? } else { item_type = (size_t)item; } } - if (item == (PyObject *)&PyLong_Type - || item == (PyObject *)&PyFloat_Type - || item == (PyObject *)&PyBool_Type) { + if (item == (PyObject *)&PyLong_Type) { PyErr_SetString(PyExc_ValueError, "unsupported type in vec[...]"); return NULL; } @@ -435,6 +442,17 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI16_Box(v); + } else if (VecBool_Check(vec)) { + double x = VecBool_UnboxItem(item); + if (VecBool_IsUnboxError(x)) { + return NULL; + } + VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); + v = VecBool_Append(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecBool_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -524,6 +542,17 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; // TODO: decref? return VecI16_Box(v); + } else if (VecBool_Check(vec)) { + char x = VecBool_UnboxItem(item); + if (VecBool_IsUnboxError(x)) { + return NULL; + } + VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); + v = VecBool_Remove(v, x); + if (VEC_IS_ERROR(v)) + return NULL; // TODO: decref? + return VecBool_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; if (!VecT_ItemCheck(v, item, v.buf->item_type)) { @@ -606,6 +635,15 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if ((result_item0 = VecI16_Box(r.f0)) == NULL) return NULL; result_item1 = VecI16_BoxItem(r.f1); + } else if (VecBool_Check(vec)) { + VecBool v = ((VecBoolObject *)vec)->vec; + VecBoolPopResult r; + r = VecBool_Pop(v, index); + if (VEC_IS_ERROR(r.f0)) + return NULL; + if ((result_item0 = VecBool_Box(r.f0)) == NULL) + return NULL; + result_item1 = VecBool_BoxItem(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; VecTPopResult r; @@ -681,6 +719,7 @@ static VecCapsule Capsule = { &I16Features, &U8Features, &FloatFeatures, + &BoolFeatures, }; PyMODINIT_FUNC @@ -747,6 +786,10 @@ PyInit_vecs(void) return NULL; if (PyType_Ready(&VecFloatBufType) < 0) return NULL; + if (PyType_Ready(&VecBoolType) < 0) + return NULL; + if (PyType_Ready(&VecBoolBufType) < 0) + return NULL; PyObject *m = PyModule_Create(&vecsmodule); if (m == NULL) diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/vecs.h index 3e9b30a27675f..1cf6fbd647287 100644 --- a/mypyc/lib-rt/vecs/vecs.h +++ b/mypyc/lib-rt/vecs/vecs.h @@ -18,6 +18,7 @@ #define VEC_ITEM_TYPE_I16 10 #define VEC_ITEM_TYPE_U8 14 #define VEC_ITEM_TYPE_FLOAT 18 +#define VEC_ITEM_TYPE_BOOL 22 inline size_t Vec_IsMagicItemType(size_t item_type) { return item_type & 2; @@ -57,6 +58,12 @@ typedef struct _VecFloatBufObject { double items[1]; } VecFloatBufObject; +// vecbuf[bool] +typedef struct _VecBoolBufObject { + PyObject_VAR_HEAD + char items[1]; +} VecBoolBufObject; + // Simple generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecTBufObject { PyObject_VAR_HEAD @@ -110,6 +117,11 @@ typedef struct _VecFloat { VecFloatBufObject *buf; } VecFloat; +typedef struct _VecBool { + Py_ssize_t len; + VecBoolBufObject *buf; +} VecBool; + typedef struct _VecT { Py_ssize_t len; VecTBufObject *buf; @@ -160,6 +172,12 @@ typedef struct _VecFloatObject { VecFloat vec; } VecFloatObject; +// Boxed vec[bool] +typedef struct _VecBoolObject { + PyObject_HEAD + VecBool vec; +} VecBoolObject; + // Simple boxed generic vecbuf: vecbuf[t] when t is a type object typedef struct _VecTObject { PyObject_HEAD @@ -218,11 +236,21 @@ typedef struct tuple_T2VFF { static tuple_T2VFF tuple_undefined_T2VFF = { { -1, NULL } , 0.0 }; #endif +#ifndef MYPYC_DECLARED_tuple_T2VCC +#define MYPYC_DECLARED_tuple_T2VCC +typedef struct tuple_T2VCC { + VecBool f0; + char f1; +} tuple_T2VCC; +static tuple_T2VCC tuple_undefined_T2VCC = { { -1, NULL } , 0 }; +#endif + typedef tuple_T2V88 VecI64PopResult; typedef tuple_T2V44 VecI32PopResult; typedef tuple_T2V22 VecI16PopResult; typedef tuple_T2VU1U1 VecU8PopResult; typedef tuple_T2VFF VecFloatPopResult; +typedef tuple_T2VCC VecBoolPopResult; // vec[i64] operations + type objects (stored in a capsule) typedef struct _VecI64Features { @@ -319,6 +347,25 @@ typedef struct _VecFloatFeatures { // iter? } VecFloatFeatures; +// vec[bool] operations + type objects (stored in a capsule) +typedef struct _VecBoolFeatures { + PyTypeObject *boxed_type; + PyTypeObject *buf_type; + VecBool (*alloc)(Py_ssize_t, Py_ssize_t); + PyObject *(*box)(VecBool); + VecBool (*unbox)(PyObject *); + VecBool (*convert_from_nested)(VecNestedBufItem); + VecBool (*append)(VecBool, char); + VecBoolPopResult (*pop)(VecBool, Py_ssize_t); + VecBool (*remove)(VecBool, char); + // TODO: Py_ssize_t + VecBool (*slice)(VecBool, int64_t, int64_t); + // PyObject *(*extend)(PyObject *, PyObject *); + // PyObject *(*concat)(PyObject *, PyObject *); + // bool (*contains)(PyObject *, char); + // iter? +} VecBoolFeatures; + #ifndef MYPYC_DECLARED_tuple_T2VOO #define MYPYC_DECLARED_tuple_T2VOO typedef struct tuple_T2VOO { @@ -390,6 +437,7 @@ typedef struct { VecI16Features *i16; VecU8Features *u8; VecFloatFeatures *float_; + VecBoolFeatures *bool_; } VecCapsule; #define BUF_SIZE(b) ((b)->ob_base.ob_size) @@ -407,6 +455,7 @@ extern PyTypeObject VecI32BufType; extern PyTypeObject VecI16BufType; extern PyTypeObject VecU8BufType; extern PyTypeObject VecFloatBufType; +extern PyTypeObject VecBoolBufType; extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; @@ -415,6 +464,7 @@ extern PyTypeObject VecI32Type; extern PyTypeObject VecI16Type; extern PyTypeObject VecU8Type; extern PyTypeObject VecFloatType; +extern PyTypeObject VecBoolType; extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; @@ -428,6 +478,7 @@ extern VecI32Features I32Features; extern VecI16Features I16Features; extern VecU8Features U8Features; extern VecFloatFeatures FloatFeatures; +extern VecBoolFeatures BoolFeatures; extern VecTFeatures TFeatures; extern VecNestedFeatures TExtFeatures; @@ -580,6 +631,42 @@ VecFloat VecFloat_Append(VecFloat, double x); VecFloat VecFloat_Remove(VecFloat, double x); VecFloatPopResult VecFloat_Pop(VecFloat v, Py_ssize_t index); +// vec[bool] operations + +static inline int VecBool_Check(PyObject *o) { + return o->ob_type == &VecBoolType; +} + +static inline PyObject *VecBool_BoxItem(char x) { + if (x == 1) { + Py_INCREF(Py_True); + return Py_True; + } else { + Py_INCREF(Py_False); + return Py_False; + } +} + +static inline char VecBool_UnboxItem(PyObject *o) { + if (o == Py_False) { + return 0; + } else if (o == Py_True) { + return 1; + } else { + PyErr_SetString(PyExc_TypeError, "bool value expected"); + return 2; + } +} + +static inline int VecBool_IsUnboxError(char x) { + return x == 2; +} + +PyObject *VecBool_Box(VecBool); +VecBool VecBool_Append(VecBool, char x); +VecBool VecBool_Remove(VecBool, char x); +VecBoolPopResult VecBool_Pop(VecBool v, Py_ssize_t index); + // vec[t] operations static inline int VecT_Check(PyObject *o) { @@ -655,6 +742,11 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem unboxed->len = o->vec.len; unboxed->buf = (PyObject *)o->vec.buf; return 0; + } else if (item->ob_type == &VecBoolType && v.buf->item_type == VEC_ITEM_TYPE_BOOL) { + VecBoolObject *o = (VecBoolObject *)item; + unboxed->len = o->vec.len; + unboxed->buf = (PyObject *)o->vec.buf; + return 0; } } else if (item->ob_type == &VecNestedType) { VecNestedObject *o = (VecNestedObject *)item; @@ -696,6 +788,9 @@ static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { } else if (item_type == VEC_ITEM_TYPE_I16) { VecI16 v = { .len = item.len, .buf = (VecI16BufObject *)item.buf }; return VecI16_Box(v); + } else if (item_type == VEC_ITEM_TYPE_BOOL) { + VecBool v = { .len = item.len, .buf = (VecBoolBufObject *)item.buf }; + return VecBool_Box(v); } else { // Generic vec[t] VecT v = { .len = item.len, .buf = (VecTBufObject *)item.buf }; From 281ea023f37b0642130ea2e59cc3177081f39133 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 13:26:30 +0000 Subject: [PATCH 144/208] Rename vecs.c to librt_vecs.c --- mypyc/lib-rt/vecs/{vecs.c => librt_vecs.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mypyc/lib-rt/vecs/{vecs.c => librt_vecs.c} (100%) diff --git a/mypyc/lib-rt/vecs/vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c similarity index 100% rename from mypyc/lib-rt/vecs/vecs.c rename to mypyc/lib-rt/vecs/librt_vecs.c From d244553c3440220d340a52cea4026a44a35e87d4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 13:29:51 +0000 Subject: [PATCH 145/208] Rename vecs.h to librt_vecs.h --- mypyc/lib-rt/vecs/librt_vecs.c | 2 +- mypyc/lib-rt/vecs/{vecs.h => librt_vecs.h} | 0 mypyc/lib-rt/vecs/vec_nested.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_template.c | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename mypyc/lib-rt/vecs/{vecs.h => librt_vecs.h} (100%) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index b9a56a1daf7b3..308f1b061c9c1 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -1,6 +1,6 @@ #define PY_SSIZE_T_CLEAN #include -#include "vecs.h" +#include "librt_vecs.h" PyTypeObject *I64TypeObj; diff --git a/mypyc/lib-rt/vecs/vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h similarity index 100% rename from mypyc/lib-rt/vecs/vecs.h rename to mypyc/lib-rt/vecs/librt_vecs.h diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 7db16a228c4ce..2557c0c892062 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -7,7 +7,7 @@ #define PY_SSIZE_T_CLEAN #include -#include "vecs.h" +#include "librt_vecs.h" static inline VecNested vec_error() { VecNested v = { .len = -1 }; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 1e34735306234..bab74373c51bb 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -9,7 +9,7 @@ #define PY_SSIZE_T_CLEAN #include -#include "vecs.h" +#include "librt_vecs.h" static inline VecT vec_error() { VecT v = { .len = -1 }; diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index c9c0397a2061d..f9f961380a647 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -13,7 +13,7 @@ #define PY_SSIZE_T_CLEAN #include -#include "vecs.h" +#include "librt_vecs.h" inline static VEC vec_error() { VEC v = { .len = -1 }; From cd826512eda21787afdc201a88ff7aebf9156acd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 14:05:53 +0000 Subject: [PATCH 146/208] Add librt.vecs capsule dep --- mypyc/ir/deps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/ir/deps.py b/mypyc/ir/deps.py index 1ca709748eeff..7c52520e274b9 100644 --- a/mypyc/ir/deps.py +++ b/mypyc/ir/deps.py @@ -48,6 +48,7 @@ def get_header(self) -> str: LIBRT_STRINGS: Final = Capsule("librt.strings") LIBRT_BASE64: Final = Capsule("librt.base64") +LIBRT_VECS: Final = Capsule("librt.vecs") BYTES_EXTRA_OPS: Final = SourceDep("bytes_extra_ops.c") BYTES_WRITER_EXTRA_OPS: Final = SourceDep("byteswriter_extra_ops.c") From 310325838febc130f639b5952fcaf9c31afaa139 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 14:06:31 +0000 Subject: [PATCH 147/208] Build librt.vecs --- mypyc/build.py | 19 +++++++++++++++++++ mypyc/lib-rt/setup.py | 16 ++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/mypyc/build.py b/mypyc/build.py index ef5d00fca1457..2f2709ae6e55f 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -104,6 +104,25 @@ class ModDesc(NamedTuple): ], ["base64"], ), + ModDesc( + "librt.vecs", + [ + "vecs/librt_vecs.c", + "vecs/vec_i64.c", + "vecs/vec_i32.c", + "vecs/vec_i16.c", + "vecs/vec_u8.c", + "vecs/vec_float.c", + "vecs/vec_bool.c", + "vecs/vec_t.c", + "vecs/vec_nested.c", + ], + [ + "vecs/librt_vecs.h", + "vecs/vec_template.c", + ], + ["vecs"], + ), ] try: diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index 964b56e4b5623..ee2cc3bcdd93c 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -132,5 +132,21 @@ def run(self) -> None: include_dirs=[".", "base64"], extra_compile_args=cflags, ), + Extension( + "librt.vecs", + [ + "vecs/librt_vecs.c", + "vecs/vec_i64.c", + "vecs/vec_i32.c", + "vecs/vec_i16.c", + "vecs/vec_u8.c", + "vecs/vec_float.c", + "vecs/vec_bool.c", + "vecs/vec_t.c", + "vecs/vec_nested.c", + ], + include_dirs=[".", "vecs"], + extra_compile_args=cflags, + ), ] ) From 6bc280b6fea5f3207e324ada5da5b04a3105ddb7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 16:31:57 +0000 Subject: [PATCH 148/208] Add librt.vecs stub --- mypy/typeshed/stubs/librt/librt/vecs.pyi | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 mypy/typeshed/stubs/librt/librt/vecs.pyi diff --git a/mypy/typeshed/stubs/librt/librt/vecs.pyi b/mypy/typeshed/stubs/librt/librt/vecs.pyi new file mode 100644 index 0000000000000..c181da627d0ff --- /dev/null +++ b/mypy/typeshed/stubs/librt/librt/vecs.pyi @@ -0,0 +1,21 @@ +from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple +from mypy_extensions import i64 + +T = TypeVar("T") + +class vec(Generic[T]): + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, __items: Iterable[T]) -> None: ... + def __len__(self) -> i64: ... + @overload + def __getitem__(self, i: i64) -> T: ... + @overload + def __getitem__(self, i: slice) -> vec[T]: ... + def __setitem__(self, i: i64, o: T) -> None: ... + def __iter__(self) -> Iterator[T]: ... + +def append(__v: vec[T], __o: T) -> vec[T]: ... +def remove(__v: vec[T], __o: T) -> vec[T]: ... +def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... From 0575c8eee9cd4a3b43ad67f3be0b91982dd63498 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 16:34:24 +0000 Subject: [PATCH 149/208] Modernize vecs stub --- mypy/typeshed/stubs/librt/librt/vecs.pyi | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypy/typeshed/stubs/librt/librt/vecs.pyi b/mypy/typeshed/stubs/librt/librt/vecs.pyi index c181da627d0ff..a4e8c88e0e596 100644 --- a/mypy/typeshed/stubs/librt/librt/vecs.pyi +++ b/mypy/typeshed/stubs/librt/librt/vecs.pyi @@ -1,4 +1,4 @@ -from typing import TypeVar, Generic, Type, Iterable, Iterator, overload, Tuple +from typing import TypeVar, Generic, Iterable, Iterator, overload from mypy_extensions import i64 T = TypeVar("T") @@ -7,15 +7,15 @@ class vec(Generic[T]): @overload def __init__(self) -> None: ... @overload - def __init__(self, __items: Iterable[T]) -> None: ... + def __init__(self, items: Iterable[T], /) -> None: ... def __len__(self) -> i64: ... @overload - def __getitem__(self, i: i64) -> T: ... + def __getitem__(self, i: i64, /) -> T: ... @overload - def __getitem__(self, i: slice) -> vec[T]: ... - def __setitem__(self, i: i64, o: T) -> None: ... + def __getitem__(self, i: slice, /) -> vec[T]: ... + def __setitem__(self, i: i64, o: T, /) -> None: ... def __iter__(self) -> Iterator[T]: ... -def append(__v: vec[T], __o: T) -> vec[T]: ... -def remove(__v: vec[T], __o: T) -> vec[T]: ... -def pop(__v: vec[T], __i: i64 = -1) -> Tuple[vec[T], T]: ... +def append(v: vec[T], o: T, /) -> vec[T]: ... +def remove(v: vec[T], o: T, /) -> vec[T]: ... +def pop(v: vec[T], i: i64 = -1, /) -> tuple[vec[T], T]: ... From 7e22214f3aea4ac7e887a671d7ab659efab83bf4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 16:41:29 +0000 Subject: [PATCH 150/208] Fix build --- mypyc/lib-rt/vecs/librt_vecs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 1cf6fbd647287..ce21446fcc2cc 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -20,7 +20,7 @@ #define VEC_ITEM_TYPE_FLOAT 18 #define VEC_ITEM_TYPE_BOOL 22 -inline size_t Vec_IsMagicItemType(size_t item_type) { +static inline size_t Vec_IsMagicItemType(size_t item_type) { return item_type & 2; } From 8d0d770db3548b662fc57e650911917f2f9ebb6f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 16:51:25 +0000 Subject: [PATCH 151/208] WIP initial vec[i64] run tests --- mypyc/test-data/run-vecs-i64.test | 342 ++++++++++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 2 files changed, 343 insertions(+) create mode 100644 mypyc/test-data/run-vecs-i64.test diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64.test new file mode 100644 index 0000000000000..1f28f324e0891 --- /dev/null +++ b/mypyc/test-data/run-vecs-i64.test @@ -0,0 +1,342 @@ +[case testLibrtVecsI64_librt_experimental] +# Test cases for vec[i64], using generic operations (simulates use from interpreted code). +# +# These also act as test cases for specialized vec types in general (e.g. vec[float]), +# since much of the implementation is shared between all of them. + +import sys +from typing import Any + +import librt.vecs +from mypy_extensions import i64 + +from testutil import assertRaises + +# Access via Any references to force generic operations +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") + +def test_vec_indexing() -> None: + assert type(vec[i64]) is type + assert type(vec[i64]) is type(vec[i64]) + assert vec[i64] is not vec + +def test_type_str() -> None: + assert str(vec[i64]) == "" + +def test_construct_empty() -> None: + v = vec[i64]() + assert type(v) is vec[i64] + assert len(v) == 0 + +def test_construct_from_initializer() -> None: + v = vec[i64]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i64](x + 1 for x in [5, 7, -8, 10]) + assert len(v) == 4 + assert v[0] == 6 + assert v[1] == 8 + assert v[2] == -7 + assert v[3] == 11 + with assertRaises(TypeError): + vec[i64](1) + with assertRaises(TypeError): + vec[i64](['1']) + with assertRaises(TypeError): + vec[i64]([None]) + +def test_append() -> None: + v = vec[i64]() + v = append(v, 1) + assert str(v) == "vec[i64]([1])" + for n in range(2, 10): + v = append(v, n) + assert str(v) == "vec[i64]([1, 2, 3, 4, 5, 6, 7, 8, 9])" + v = vec[i64]() + v = append(v, 2**63 - 1) + v = append(v, -2**63) + assert v[0] == 2**63 - 1 + assert v[1] == -2**63 + +def test_append_error() -> None: + v: Any = vec[i64]() + with assertRaises(OverflowError): + v = append(v, 2**63) + with assertRaises(OverflowError): + v = append(v, -2**63 - 1) + assert len(v) == 0 + with assertRaises(TypeError): + v = append(v, 'x') + with assertRaises(TypeError): + v = append(v, 1.5) + assert len(v) == 0 + +def test_repr() -> None: + assert str(vec[i64]()) == "vec[i64]([])" + +def test_size() -> None: + v = vec[i64]() + assert getsizeof(v) == 32 + v = append(v, 123) + assert getsizeof(v) == 32 + +def test_get_item() -> None: + v = append(vec[i64](), 44) + assert v[0] == 44 + v = append(v, 56) + assert v[0] == 44 + assert v[1] == 56 + +def test_get_item_error() -> None: + v = append(vec[i64](), 1) + v = append(v, 2) + v = append(v, 3) + with assertRaises(IndexError): + v[3] + with assertRaises(IndexError): + v[-4] + n = 2 ** 100 + with assertRaises(IndexError): + v[n] + +def test_get_item_negative() -> None: + v = vec[i64]([2, 5, 7]) + assert v[-1] == 7 + assert v[-2] == 5 + assert v[-3] == 2 + with assertRaises(IndexError): + v[-4] + with assertRaises(IndexError): + v[-2**62] + +def test_len() -> None: + v = vec[i64]() + assert len(v) == 0 + v = append(v, 1) + assert len(v) == 1 + v = append(v, 2) + assert len(v) == 2 + for i in range(100): + v = append(v, i) + assert len(v) == i + 3 + +def test_set_item() -> None: + v = vec[i64]() + v = append(v, 1) + v = append(v, 2) + v[0] = 3 + assert v[0] == 3 + assert v[1] == 2 + v[1] = 5 + assert v[0] == 3 + assert v[1] == 5 + +def test_set_item_error() -> None: + v = append(vec[i64](), 1) + v = append(v, 2) + v = append(v, 3) + with assertRaises(IndexError): + v[3] = 1 + v[0] = 2**63 - 1 + assert v[0] == 2**63 - 1 + with assertRaises(OverflowError): + v[0] = 2**63 + v[0] = -2**63 + assert v[0] == -2**63 + with assertRaises(OverflowError): + v[0] = -2**63 - 1 + with assertRaises(TypeError): + v[0] = "x" + with assertRaises(TypeError): + v[0] = 1.5 + +def test_set_item_negative() -> None: + v = vec[i64]() + v = append(v, 1) + v = append(v, 2) + v[-2] = 3 + assert v[0] == 3 + assert v[1] == 2 + v[-1] = 5 + assert v[0] == 3 + assert v[1] == 5 + with assertRaises(IndexError): + v[-3] = 1 + +def test_equality() -> None: + assert_eq(vec[i64](), vec[i64]()) + assert_eq(vec[i64]([1]), vec[i64]([1])) + v = vec[i64]([1]) + assert_eq(v, v) + assert_eq(vec[i64]([1, 2]), vec[i64]([1, 2])) + + assert_neq(vec[i64](), vec[i64]([1])) + assert_neq(vec[i64]([1, 2]), vec[i64]([1])) + assert_neq(vec[i64]([1]), vec[i64]([2])) + assert_neq(vec[i64]([1, 2]), vec[i64]([2, 2])) + assert_neq(vec[i64]([1, 2]), vec[i64]([1, 1])) + +if False: # TODO + def test_equality_different_types() -> None: + assert_neq(vec[i64]([1]), [1]) + assert_neq(vec[i64]([1]), (1,)) + assert_neq(vec[i64]([1]), vec[object]([1])) + assert_neq(vec[i64](), vec[object]()) + +def assert_eq(x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b + +def assert_neq(x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + +def test_iteration() -> None: + for x in vec[i64](): + assert False + a = [] + for x in vec[i64]([11]): + a.append(x) + assert a == [11] + a = [] + for x in vec[i64]([11, 23, 45, 64]): + a.append(x) + assert a == [11, 23, 45, 64] + +def test_iteration_nested() -> None: + v = vec[i64]() + v = append(v, 2) + v = append(v, 5) + v = append(v, 7) + a = [] + for x in v: + for y in v: + a.append((x, y)) + assert a == [(2, 2), (2, 5), (2, 7), + (5, 2), (5, 5), (5, 7), + (7, 2), (7, 5), (7, 7)] + +def test_slicing() -> None: + v = vec[i64](range(5)) + assert v[1:4] == vec[i64]([1, 2, 3]) + assert v[2:-1] == vec[i64]([2, 3]) + assert v[-2:-1] == vec[i64]([3]) + assert v[1:] == vec[i64]([1, 2, 3, 4]) + assert v[:-1] == vec[i64]([0, 1, 2, 3]) + assert v[:] == v + assert v[:] is not v + assert v[5:] == vec[i64]() + assert v[100:] == vec[i64]() + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[i64]([2, 3, 4]) + assert v[-100:2] == vec[i64]([0, 1]) + assert v[5:100] == vec[i64]([]) + assert v[50:100] == vec[i64]([]) + assert v[-100:-50] == vec[i64]([]) + +def test_slicing_with_step() -> None: + v = vec[i64](range(10)) + assert v[1:5:2] == vec[i64]([1, 3]) + assert v[1:6:2] == vec[i64]([1, 3, 5]) + assert v[5:1:-2] == vec[i64]([5, 3]) + assert v[6:1:-2] == vec[i64]([6, 4, 2]) + v = vec[i64](range(5)) + assert v[::-1] == vec[i64]([4, 3, 2, 1, 0]) + with assertRaises(ValueError): + v[1:3:0] + +def test_contains() -> None: + v = vec[i64](range(2, 7)) + for i in range(2, 7): + assert i in v + assert 1 not in v + assert 7 not in v + assert -1 not in v + assert 1233453452 not in v + assert -1233453452 not in v + vv = vec[i64]() + assert 3 not in vv + +def test_remove() -> None: + a = [4, 7, 9] + for i in a: + v = vec[i64](a) + v = remove(v, i) + assert v == vec[i64]([j for j in a if j != i]) + v = vec[i64](a) + v = remove(v, 4) + v = remove(v, 7) + v = remove(v, 9) + assert v == vec[i64]() + with assertRaises(ValueError): + remove(v, 4) + v = append(v, 5) + with assertRaises(ValueError): + remove(v, 7) + v = remove(v, 5) + assert len(v) == 0 + v = vec[i64]([1, 1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1, 1]) + v = remove(v, 1) + assert v == vec[i64]([1]) + v = remove(v, 1) + assert v == vec[i64]() + f: Any = 1.1 + with assertRaises(TypeError): + remove(v, f) + s: Any = 'x' + with assertRaises(TypeError): + remove(v, s) + +def test_pop_last() -> None: + v = vec[i64]([4, 7, 9]) + v, n = pop(v) + assert n == 9 + assert v == vec[i64]([4, 7]) + v, n = pop(v) + assert n == 7 + assert v == vec[i64]([4]) + v, n = pop(v) + assert n == 4 + assert v == vec[i64]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[i64]([4, 7, 9, 15, 22]) + v, n = pop(v, 0) + assert n == 4 + assert v == vec[i64]([7, 9, 15, 22]) + v, n = pop(v, -1) + assert n == 22 + assert v == vec[i64]([7, 9, 15]) + v, n = pop(v, 1) + assert n == 9 + assert v == vec[i64]([7, 15]) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, n = pop(v, -2) + assert n == 7 + assert v == vec[i64]([15]) + v, n = pop(v, 0) + assert n == 15 + assert v == vec[i64]() diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 8157d741bf084..a1cd87fdb83d2 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -78,6 +78,7 @@ "run-librt-strings.test", "run-base64.test", "run-match.test", + "run-vecs-i64.test", ] if sys.version_info >= (3, 12): From bc9da90023b204a68fa4df30d9ad34b254061e84 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 18:07:15 +0000 Subject: [PATCH 152/208] Fix tests --- mypyc/test-data/run-vecs-i64.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64.test index 1f28f324e0891..9109a51528382 100644 --- a/mypyc/test-data/run-vecs-i64.test +++ b/mypyc/test-data/run-vecs-i64.test @@ -8,7 +8,7 @@ import sys from typing import Any import librt.vecs -from mypy_extensions import i64 +import mypy_extensions from testutil import assertRaises @@ -18,6 +18,7 @@ append: Any = librt.vecs.append remove: Any = librt.vecs.remove pop: Any = librt.vecs.pop getsizeof: Any = getattr(sys, "getsizeof") +i64: Any = mypy_extensions.i64 def test_vec_indexing() -> None: assert type(vec[i64]) is type From 2114b6e4b13af3dfae30310ff9d04fb5c2d09f4b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Wed, 21 Jan 2026 18:15:12 +0000 Subject: [PATCH 153/208] Improve some exception error messages --- mypyc/lib-rt/vecs/librt_vecs.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 308f1b061c9c1..951112a49cb66 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -227,7 +227,8 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) } } if (item == (PyObject *)&PyLong_Type) { - PyErr_SetString(PyExc_ValueError, "unsupported type in vec[...]"); + PyErr_Format(PyExc_ValueError, "unsupported type in vec[%s] (use i64, i32, i16, or u8)", + ((PyTypeObject *)item)->tp_name); return NULL; } VecProxy *p; @@ -474,7 +475,8 @@ static PyObject *vec_append(PyObject *self, PyObject *args) return NULL; // TODO: decref? return VecVec_Box(v); } else { - PyErr_SetString(PyExc_TypeError, "vec argument expected"); + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); return NULL; } } @@ -574,7 +576,8 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) return NULL; // TODO: decref? return VecVec_Box(v); } else { - PyErr_SetString(PyExc_TypeError, "vec argument expected"); + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); return NULL; } } @@ -675,7 +678,8 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) return NULL; } } else { - PyErr_SetString(PyExc_TypeError, "vec argument expected"); + PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", + Py_TYPE(vec)->tp_name); return NULL; } From f0030ebb886c23dfe45a41f6041c1c9161d9dec2 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 12:17:51 +0000 Subject: [PATCH 154/208] Rename capsule to librt.vecs._C_API (add librt prefix) --- mypyc/lib-rt/vecs/librt_vecs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 951112a49cb66..bc186c9feb235 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -806,7 +806,7 @@ PyInit_vecs(void) return NULL; } - PyObject *c_api = PyCapsule_New(&Capsule, "vecs._C_API", NULL); + PyObject *c_api = PyCapsule_New(&Capsule, "librt.vecs._C_API", NULL); if (c_api == NULL) return NULL; From 3e2f0b4d36f4dfe534ebefa8b9dcc37b7219a959 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 13:00:50 +0000 Subject: [PATCH 155/208] Update tests --- mypyc/test-data/run-vecs-i64.test | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64.test index 9109a51528382..9f983a1d61515 100644 --- a/mypyc/test-data/run-vecs-i64.test +++ b/mypyc/test-data/run-vecs-i64.test @@ -1,4 +1,4 @@ -[case testLibrtVecsI64_librt_experimental] +[case testLibrtVecsI64Interpreted_librt_experimental] # Test cases for vec[i64], using generic operations (simulates use from interpreted code). # # These also act as test cases for specialized vec types in general (e.g. vec[float]), @@ -62,6 +62,11 @@ def test_append() -> None: v = append(v, -2**63) assert v[0] == 2**63 - 1 assert v[1] == -2**63 + v = vec[i64]() + for i in range(10000): + v = append(v, i) + for i in range(10000): + assert v[i] == i def test_append_error() -> None: v: Any = vec[i64]() @@ -78,6 +83,8 @@ def test_append_error() -> None: def test_repr() -> None: assert str(vec[i64]()) == "vec[i64]([])" + assert str(vec[i64]([123])) == "vec[i64]([123])" + assert str(vec[i64]([123, -987654321])) == "vec[i64]([123, -987654321])" def test_size() -> None: v = vec[i64]() @@ -181,12 +188,12 @@ def test_equality() -> None: assert_neq(vec[i64]([1, 2]), vec[i64]([2, 2])) assert_neq(vec[i64]([1, 2]), vec[i64]([1, 1])) -if False: # TODO - def test_equality_different_types() -> None: - assert_neq(vec[i64]([1]), [1]) - assert_neq(vec[i64]([1]), (1,)) - assert_neq(vec[i64]([1]), vec[object]([1])) - assert_neq(vec[i64](), vec[object]()) +def test_equality_different_types() -> None: + assert_neq(vec[i64]([1]), None) + assert_neq(vec[i64]([1]), [1]) + assert_neq(vec[i64]([1]), (1,)) + assert_neq(vec[i64]([1]), vec[object]([1])) + assert_neq(vec[i64](), vec[object]()) def assert_eq(x: object, y: object) -> None: assert x == y From b14345b659917bf77bf3e352426f8fbe82b68336 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 13:47:52 +0000 Subject: [PATCH 156/208] Rename test file --- mypyc/test-data/{run-vecs-i64.test => run-vecs-i64-interp.test} | 0 mypyc/test/test_run.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename mypyc/test-data/{run-vecs-i64.test => run-vecs-i64-interp.test} (100%) diff --git a/mypyc/test-data/run-vecs-i64.test b/mypyc/test-data/run-vecs-i64-interp.test similarity index 100% rename from mypyc/test-data/run-vecs-i64.test rename to mypyc/test-data/run-vecs-i64-interp.test diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index a1cd87fdb83d2..cd31f718af097 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -78,7 +78,7 @@ "run-librt-strings.test", "run-base64.test", "run-match.test", - "run-vecs-i64.test", + "run-vecs-i64-interp.test", ] if sys.version_info >= (3, 12): From 5bd859835740cbcadf377652e9f9ef93690052d1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 13:59:22 +0000 Subject: [PATCH 157/208] Update test --- mypyc/test-data/run-vecs-i64-interp.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index 9f983a1d61515..dbb60c9e7759e 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -27,6 +27,7 @@ def test_vec_indexing() -> None: def test_type_str() -> None: assert str(vec[i64]) == "" + assert repr(vec[i64]) == "" def test_construct_empty() -> None: v = vec[i64]() From 27220dec4146dff44a280f9eb895021173c67137 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:08:47 +0000 Subject: [PATCH 158/208] Fix reference counting and add error checks --- mypyc/lib-rt/vecs/librt_vecs.c | 156 +++++++++++++++++++++++++-------- 1 file changed, 120 insertions(+), 36 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index bc186c9feb235..96a6b6a1463a4 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -69,8 +69,10 @@ VecProxy_dealloc(VecProxy *self) } PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { - PyObject *item; - if (depth == 0) + PyObject *item = NULL; + PyObject *result = NULL; + + if (depth == 0) { if ((item_type & ~1) == VEC_ITEM_TYPE_I64) { item = PyUnicode_FromFormat("i64"); } else if ((item_type & ~1) == VEC_ITEM_TYPE_U8) { @@ -85,27 +87,69 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { item = PyUnicode_FromFormat("bool"); } else { item = PyObject_GetAttrString((PyObject *)(item_type & ~1), "__name__"); + if (item == NULL) { + return NULL; + } if (item_type & 1) { - item = PyUnicode_FromFormat("%U | None", item); + PyObject *optional_item = PyUnicode_FromFormat("%U | None", item); + Py_DECREF(item); + if (optional_item == NULL) { + return NULL; + } + item = optional_item; } } - else + } else { item = Vec_TypeToStr(item_type, depth - 1); + } - return PyUnicode_FromFormat("vec[%U]", item); + if (item == NULL) { + return NULL; + } + + result = PyUnicode_FromFormat("vec[%U]", item); + Py_DECREF(item); + return result; } PyObject *VecProxy_repr(PyObject *self) { - // TODO: error handling, refcounting, etc. - PyObject *l = Py_BuildValue("[]"); - PyObject *prefix = Py_BuildValue("s", ""); - PyObject *sep = Py_BuildValue("s", ""); - PyList_Append(l, prefix); + PyObject *l = NULL; + PyObject *prefix = NULL; + PyObject *suffix = NULL; + PyObject *sep = NULL; + PyObject *type_str = NULL; + PyObject *result = NULL; + + l = PyList_New(0); + if (l == NULL) goto error; + + prefix = PyUnicode_FromString(""); + if (suffix == NULL) goto error; + + sep = PyUnicode_FromString(""); + if (sep == NULL) goto error; + + if (PyList_Append(l, prefix) < 0) goto error; + VecProxy *v = (VecProxy *)self; - PyList_Append(l, Vec_TypeToStr(v->item_type, v->depth)); - PyList_Append(l, suffix); - return PyUnicode_Join(sep, l); + type_str = Vec_TypeToStr(v->item_type, v->depth); + if (type_str == NULL) goto error; + + if (PyList_Append(l, type_str) < 0) goto error; + if (PyList_Append(l, suffix) < 0) goto error; + + result = PyUnicode_Join(sep, l); + +error: + Py_XDECREF(l); + Py_XDECREF(prefix); + Py_XDECREF(suffix); + Py_XDECREF(sep); + Py_XDECREF(type_str); + return result; } PyTypeObject VecProxyType = { @@ -258,46 +302,86 @@ PyTypeObject VecGenericType = { }; PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose) { - // TODO: Check for errors - // TODO: Refcount handling - PyObject *l = Py_BuildValue("[]"); - PyObject *prefix; - PyObject *mid; - PyObject *suffix; - PyObject *sep = Py_BuildValue("s", ""); - PyObject *comma = Py_BuildValue("s", ", "); + PyObject *l = NULL; + PyObject *prefix = NULL; + PyObject *mid = NULL; + PyObject *suffix = NULL; + PyObject *sep = NULL; + PyObject *comma = NULL; + PyObject *result = NULL; + + l = PyList_New(0); + if (l == NULL) goto error; + + sep = PyUnicode_FromString(""); + if (sep == NULL) goto error; + + comma = PyUnicode_FromString(", "); + if (comma == NULL) goto error; + if (verbose) { prefix = Vec_TypeToStr(item_type, depth); - mid = Py_BuildValue("s", "(["); - suffix = Py_BuildValue("s", "])"); + if (prefix == NULL) goto error; + + mid = PyUnicode_FromString("(["); + if (mid == NULL) goto error; + + suffix = PyUnicode_FromString("])"); + if (suffix == NULL) goto error; } else { - prefix = Py_BuildValue("s", ""); - mid = Py_BuildValue("s", "["); - suffix = Py_BuildValue("s", "]"); + prefix = PyUnicode_FromString(""); + if (prefix == NULL) goto error; + + mid = PyUnicode_FromString("["); + if (mid == NULL) goto error; + + suffix = PyUnicode_FromString("]"); + if (suffix == NULL) goto error; } - PyList_Append(l, prefix); - PyList_Append(l, mid); + + if (PyList_Append(l, prefix) < 0) goto error; + if (PyList_Append(l, mid) < 0) goto error; Py_ssize_t len = PyObject_Length(vec); + if (len < 0) goto error; for (Py_ssize_t i = 0; i < len; i++) { PyObject *it = PySequence_GetItem(vec, i); + if (it == NULL) goto error; + PyObject *r; if (depth == 0 || it == Py_None) { r = PyObject_Repr(it); } else { r = Vec_GenericRepr(it, item_type, depth - 1, 0); } - if (r == NULL) - return NULL; - PyList_Append(l, r); - if (i + 1 < len) - PyList_Append(l, comma); + Py_DECREF(it); + + if (r == NULL) goto error; + + if (PyList_Append(l, r) < 0) { + Py_DECREF(r); + goto error; + } + Py_DECREF(r); + + if (i + 1 < len) { + if (PyList_Append(l, comma) < 0) goto error; + } } - PyList_Append(l, suffix); + if (PyList_Append(l, suffix) < 0) goto error; + + result = PyUnicode_Join(sep, l); - return PyUnicode_Join(sep, l); +error: + Py_XDECREF(l); + Py_XDECREF(prefix); + Py_XDECREF(mid); + Py_XDECREF(suffix); + Py_XDECREF(sep); + Py_XDECREF(comma); + return result; } // Generic comparison implementation for vecs with PyObject * items. From 5db0e92f1208db999e30c9a989bb25bd206acf17 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:14:19 +0000 Subject: [PATCH 159/208] WIP tests for vec[t] --- mypyc/test-data/run-vecs-t-interp.test | 440 +++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100644 mypyc/test-data/run-vecs-t-interp.test diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test new file mode 100644 index 0000000000000..2dc2ab0363e86 --- /dev/null +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -0,0 +1,440 @@ +[case testLibrtVecsI64Interpreted_librt_experimental] +# Test cases for vec[], using generic operations (simulates use from interpreted code). +import sys +from typing import cast, List, Optional + +from vecs import vec, append, remove, pop + +from mypy_extensions import i64 + + +class TestVecT(unittest.TestCase): + def test_vec_indexing(self) -> None: + assert vec[str] is not vec[i64] + assert type(vec[str]) is not type + assert type(vec[Optional[str]]) is type(vec[str]) + + def test_type_str(self) -> None: + assert str(vec[str]) == "" + assert str(vec[Optional[str]]) == "" + + def test_construct_empty(self) -> None: + v = vec[str]() + assert len(v) == 0 + v2 = vec[Optional[str]]() + assert len(v2) == 0 + assert type(v) is type(v2) + + def test_construct_or_union(self) -> None: + if sys.version_info >= (3, 10): + v = vec[str | None]() + assert len(v) == 0 + assert str(v) == 'vec[str | None]([])' + + def test_construct_from_initializer(self) -> None: + v = vec[str](['xyz']) + assert len(v) == 1 + assert v[0] == 'xyz' + v = vec[str](x + '!' for x in ['x', '', 'foo', 'bar']) + assert len(v) == 4 + assert v[0] == 'x!' + assert v[1] == '!' + assert v[2] == 'foo!' + assert v[3] == 'bar!' + with self.assertRaises(TypeError): + vec[str](1) + with self.assertRaises(TypeError): + vec[str]([1]) + with self.assertRaises(TypeError): + vec[str]([None]) + + def test_construct_from_initializer_optional(self) -> None: + v = vec[Optional[str]]([None, 'xyz']) + assert len(v) == 2 + assert v[0] is None + assert v[1] == 'xyz' + v = vec[Optional[str]](x for x in ['x', None, 'foo', 'bar']) + assert len(v) == 4 + assert v[0] == 'x' + assert v[1] is None + assert v[2] == 'foo' + assert v[3] == 'bar' + with self.assertRaises(TypeError): + vec[Optional[str]](1) + with self.assertRaises(TypeError): + vec[Optional[str]]([1]) + + def test_append(self) -> None: + v = vec[str]() + v = append(v, '1') + assert str(v) == "vec[str](['1'])" + for n in range(2, 10): + v = append(v, str(n)) + assert str(v) == "vec[str](['1', '2', '3', '4', '5', '6', '7', '8', '9'])" + + v2 = vec[Optional[str]]() + v2 = append(v2, 'x') + v2 = append(v2, None) + assert str(v2) == "vec[str | None](['x', None])" + + def test_append_error(self) -> None: + v: Any = vec[str]() + with self.assertRaises(TypeError): + v = append(v, b'x') + with self.assertRaises(TypeError): + v = append(v, ['x']) + with self.assertRaises(TypeError): + v = append(v, None) + assert len(v) == 0 + + v2: Any = vec[Optional[str]]() + with self.assertRaises(TypeError): + v2 = append(v2, b'x') + + def test_repr(self) -> None: + assert str(vec[str]()) == "vec[str]([])" + assert str(vec[Optional[str]]([None, 'x'])) == "vec[str | None]([None, 'x'])" + + def test_size(self) -> None: + v = vec[str]() + assert sys.getsizeof(v) == 48 + v = append(v, '123') + assert sys.getsizeof(v) == 48 + + def test_get_item(self) -> None: + v = append(vec[str](), '44') + assert v[0] == '44' + v = append(v, '56') + assert v[0] == '44' + assert v[1] == '56' + + v2 = vec[Optional[str]]([None, '12']) + assert v2[0] is None + assert v2[1] == '12' + + def test_get_item_error(self) -> None: + v = append(vec[str](), '1') + v = append(v, '2') + v = append(v, '3') + with self.assertRaises(IndexError): + v[3] + with self.assertRaises(IndexError): + v[-4] + n = 2 ** 100 + with self.assertRaises(IndexError): + v[n] + + def test_get_item_negative(self) -> None: + v = vec[str](['2', '5', '7']) + assert v[-1] == '7' + assert v[-2] == '5' + assert v[-3] == '2' + with self.assertRaises(IndexError): + v[-4] + with self.assertRaises(IndexError): + v[-2**62] + + def test_len(self) -> None: + v = vec[str]() + assert len(v) == 0 + v = append(v, '1') + assert len(v) == 1 + v = append(v, '2') + assert len(v) == 2 + for i in range(100): + v = append(v, 'i') + assert len(v) == i + 3 + + def test_set_item(self) -> None: + v = vec[str]() + v = append(v, '1') + v = append(v, '2') + v[0] = '3' + assert v[0] == '3' + assert v[1] == '2' + v[1] = '5' + assert v[0] == '3' + assert v[1] == '5' + + def test_set_item_optional(self) -> None: + v = vec[Optional[str]]() + v = append(v, '1') + v = append(v, None) + v[0] = '3' + assert v[0] == '3' + assert v[1] is None + v[1] = '5' + assert v[0] == '3' + assert v[1] == '5' + v[1] = None + assert v[0] == '3' + assert v[1] is None + + def test_set_item_error(self) -> None: + v = append(vec[str](), '1') + v = append(v, '2') + v = append(v, '3') + with self.assertRaises(IndexError): + v[3] = '1' + with self.assertRaises(TypeError): + v[0] = b'x' + with self.assertRaises(TypeError): + v[0] = vec[str]() + with self.assertRaises(TypeError): + v[0] = None + assert v[0] == '1' + + def test_set_item_error_optional(self) -> None: + v = append(vec[Optional[str]](), '1') + v = append(v, None) + v = append(v, '3') + with self.assertRaises(IndexError): + v[3] = '1' + with self.assertRaises(IndexError): + v[3] = None + with self.assertRaises(IndexError): + v[-4] = '1' + with self.assertRaises(IndexError): + v[-4] = None + with self.assertRaises(TypeError): + v[0] = b'x' + with self.assertRaises(TypeError): + v[0] = vec[str]() + with self.assertRaises(TypeError): + v[0] = vec[Optional[str]]() + assert v[0] == '1' + + def test_set_item_negative(self) -> None: + v = vec[str]() + v = append(v, '1') + v = append(v, '2') + v[-2] = '3' + assert v[0] == '3' + assert v[1] == '2' + v[-1] = '5' + assert v[0] == '3' + assert v[1] == '5' + with self.assertRaises(IndexError): + v[-3] = '1' + + def test_nested_in_type(self) -> None: + cast(List[vec[str]], []) + + def test_cannot_overload_isinstance(self) -> None: + class M(type): + def __instancecheck__(cls, instance): + return isinstance(instance, str) + + class C(metaclass=M): pass + + assert isinstance('', C) + v = vec[C]() + v = append(v, C()) + with self.assertRaises(TypeError): + v = append(v, '') + with self.assertRaises(TypeError): + v[0] = '' + + def test_add_subclass_instance(self) -> None: + class SS(str): + pass + + v = vec[str]() + v = append(v, SS()) + v[0] = SS() + + v2 = vec[SS]() + with self.assertRaises(TypeError): + v2 = append(v2, '') + + def test_equality(self) -> None: + self.assert_eq(vec[str](), vec[str]()) + self.assert_eq(vec[str](['x']), vec[str](['x'])) + v = vec[str](['x']) + self.assert_eq(v, v) + self.assert_eq(vec[str](['x', 'y']), vec[str](['x', 'y'])) + + self.assert_neq(vec[str](), vec[str](['x'])) + self.assert_neq(vec[str](['x', 'y']), vec[str](['x'])) + self.assert_neq(vec[str](['x']), vec[str](['y'])) + self.assert_neq(vec[str](['x', 'y']), vec[str](['y', 'y'])) + self.assert_neq(vec[str](['x', 'y']), vec[str](['x', 'x'])) + + self.assert_eq(vec[Optional[str]](), vec[Optional[str]]()) + self.assert_eq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', None])) + self.assert_neq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', 'y'])) + + def test_equality_different_types(self) -> None: + self.assert_neq(vec[str](['x']), ['x']) + self.assert_neq(vec[str](['x']), ('x',)) + self.assert_neq(vec[str](['x']), vec[object](['x'])) + self.assert_neq(vec[str](), vec[object]()) + self.assert_neq(vec[str](), vec[Optional[str]]()) + + def test_exception_from_eq(self) -> None: + class C: + def __eq__(self, other): + raise RuntimeError() + + with self.assertRaises(RuntimeError): + vec[C]([C()]) == vec[C]([C()]) + + def assert_eq(self, x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b + + def assert_neq(self, x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + + def test_iteration(self) -> None: + for x in vec[str](): + assert False + a = [] + for x in vec[str](['x']): + a.append(x) + assert a == ['x'] + a = [] + for x in vec[str](['11', '23', '45', '64']): + a.append(x) + assert a == ['11', '23', '45', '64'] + a = [] + for x in vec[Optional[str]](['11', None, '45', '64']): + a.append(x) + assert a == ['11', None, '45', '64'] + + def test_slicing(self) -> None: + v = vec[str](str(i) for i in range(5)) + assert v[1:4] == vec[str](['1', '2', '3']) + assert v[2:-1] == vec[str](['2', '3']) + assert v[-2:-1] == vec[str](['3']) + assert v[1:] == vec[str](['1', '2', '3', '4']) + assert v[:-1] == vec[str](['0', '1', '2', '3']) + assert v[:] == v + assert v[:] is not v + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[str](['2', '3', '4']) + assert v[-100:2] == vec[str](['0', '1']) + assert v[5:100] == vec[str]([]) + assert v[50:100] == vec[str]([]) + assert v[-100:-50] == vec[str]([]) + + def test_slicing_optional(self) -> None: + v = vec[Optional[str]](['x', None, 'y']) + assert v[:2] == vec[Optional[str]](['x', None]) + + def test_slicing_with_step(self) -> None: + v = vec[str](str(i) for i in range(10)) + assert v[1:5:2] == vec[str](['1', '3']) + assert v[1:6:2] == vec[str](['1', '3', '5']) + assert v[5:1:-2] == vec[str](['5', '3']) + assert v[6:1:-2] == vec[str](['6', '4', '2']) + v = vec[str](str(i) for i in range(5)) + assert v[::-1] == vec[str](['4', '3', '2', '1', '0']) + with self.assertRaises(ValueError): + v[1:3:0] + + def test_contains(self) -> None: + v = vec[str](str(i) for i in range(2, 7)) + for i in range(2, 7): + assert str(i) in v + assert 'x' not in v + assert '22' not in v + vv = vec[str]() + assert '3' not in vv + + v2 = vec[Optional[str]]() + assert '3' not in v2 + assert None not in v2 + v2 = vec[Optional[str]](['3', None]) + assert '3' in v2 + assert None in v2 + assert '4' not in v2 + + def test_remove(self) -> None: + a = ['4x', '7x', '9x'] + for i in a: + v = vec[str](a) + v = remove(v, i) + assert v == vec[str]([j for j in a if j != i]) + v = vec[str](a) + # Make sure we have a different id + v = remove(v, (i + 'x')[:-1]) + assert v == vec[str]([j for j in a if j != i]) + v = vec[str](a) + v = remove(v, '4x') + v = remove(v, '7x') + v = remove(v, '9x') + assert v == vec[str]() + with self.assertRaises(ValueError): + remove(v, '4') + v = append(v, '5') + with self.assertRaises(ValueError): + remove(v, '7') + v = remove(v, '5') + assert len(v) == 0 + v = vec[str](['x', 'x', 'x']) + v = remove(v, 'x') + assert v == vec[str](['x', 'x']) + v = remove(v, 'x') + assert v == vec[str](['x']) + v = remove(v, 'x') + assert v == vec[str]() + b: Any = b'x' + with self.assertRaises(TypeError): + remove(v, b) + + def test_remove_optional(self) -> None: + v = vec[Optional[str]](['x', None]) + v = remove(v, 'x') + assert str(v) == "vec[str | None]([None])" + v = remove(v, None) + assert str(v) == "vec[str | None]([])" + + def test_pop_last(self) -> None: + v = vec[str](['4', '7', '9']) + v, item = pop(v) + assert item == '9' + assert v == vec[str](['4', '7']) + v, item = pop(v) + assert item == '7' + assert v == vec[str](['4']) + v, item = pop(v) + assert item == '4' + assert v == vec[str]() + with self.assertRaises(IndexError): + pop(v) + + def test_pop_index(self) -> None: + v = vec[str](['4', '7', '9', '15', '22']) + v, item = pop(v, 0) + assert item == '4' + assert v == vec[str](['7', '9', '15', '22']) + v, item = pop(v, -1) + assert item == '22' + assert v == vec[str](['7', '9', '15']) + v, item = pop(v, 1) + assert item == '9' + assert v == vec[str](['7', '15']) + + with self.assertRaises(IndexError): + pop(v, 2) + + with self.assertRaises(IndexError): + pop(v, -3) + + v, item = pop(v, -2) + assert item == '7' + assert v == vec[str](['15']) + v, item = pop(v, 0) + assert item == '15' + assert v == vec[str]() From e5d11b6e6c42afcb37afb836c0356f26017d0ece Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:21:08 +0000 Subject: [PATCH 160/208] Update --- mypyc/test-data/run-vecs-t-interp.test | 896 +++++++++++++------------ 1 file changed, 469 insertions(+), 427 deletions(-) diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test index 2dc2ab0363e86..e17a2e2d2e926 100644 --- a/mypyc/test-data/run-vecs-t-interp.test +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -1,440 +1,482 @@ [case testLibrtVecsI64Interpreted_librt_experimental] # Test cases for vec[], using generic operations (simulates use from interpreted code). import sys -from typing import cast, List, Optional +from typing import cast, List, Optional, Any -from vecs import vec, append, remove, pop +import librt.vecs +import mypy_extensions -from mypy_extensions import i64 +from testutil import assertRaises +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") +i64: Any = mypy_extensions.i64 -class TestVecT(unittest.TestCase): - def test_vec_indexing(self) -> None: - assert vec[str] is not vec[i64] - assert type(vec[str]) is not type - assert type(vec[Optional[str]]) is type(vec[str]) - def test_type_str(self) -> None: - assert str(vec[str]) == "" - assert str(vec[Optional[str]]) == "" +def assert_eq(x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b - def test_construct_empty(self) -> None: - v = vec[str]() - assert len(v) == 0 - v2 = vec[Optional[str]]() - assert len(v2) == 0 - assert type(v) is type(v2) - - def test_construct_or_union(self) -> None: - if sys.version_info >= (3, 10): - v = vec[str | None]() - assert len(v) == 0 - assert str(v) == 'vec[str | None]([])' - - def test_construct_from_initializer(self) -> None: - v = vec[str](['xyz']) - assert len(v) == 1 - assert v[0] == 'xyz' - v = vec[str](x + '!' for x in ['x', '', 'foo', 'bar']) - assert len(v) == 4 - assert v[0] == 'x!' - assert v[1] == '!' - assert v[2] == 'foo!' - assert v[3] == 'bar!' - with self.assertRaises(TypeError): - vec[str](1) - with self.assertRaises(TypeError): - vec[str]([1]) - with self.assertRaises(TypeError): - vec[str]([None]) - - def test_construct_from_initializer_optional(self) -> None: - v = vec[Optional[str]]([None, 'xyz']) - assert len(v) == 2 - assert v[0] is None - assert v[1] == 'xyz' - v = vec[Optional[str]](x for x in ['x', None, 'foo', 'bar']) - assert len(v) == 4 - assert v[0] == 'x' - assert v[1] is None - assert v[2] == 'foo' - assert v[3] == 'bar' - with self.assertRaises(TypeError): - vec[Optional[str]](1) - with self.assertRaises(TypeError): - vec[Optional[str]]([1]) - - def test_append(self) -> None: - v = vec[str]() - v = append(v, '1') - assert str(v) == "vec[str](['1'])" - for n in range(2, 10): - v = append(v, str(n)) - assert str(v) == "vec[str](['1', '2', '3', '4', '5', '6', '7', '8', '9'])" - - v2 = vec[Optional[str]]() - v2 = append(v2, 'x') - v2 = append(v2, None) - assert str(v2) == "vec[str | None](['x', None])" - - def test_append_error(self) -> None: - v: Any = vec[str]() - with self.assertRaises(TypeError): - v = append(v, b'x') - with self.assertRaises(TypeError): - v = append(v, ['x']) - with self.assertRaises(TypeError): - v = append(v, None) - assert len(v) == 0 - v2: Any = vec[Optional[str]]() - with self.assertRaises(TypeError): - v2 = append(v2, b'x') - - def test_repr(self) -> None: - assert str(vec[str]()) == "vec[str]([])" - assert str(vec[Optional[str]]([None, 'x'])) == "vec[str | None]([None, 'x'])" - - def test_size(self) -> None: - v = vec[str]() - assert sys.getsizeof(v) == 48 - v = append(v, '123') - assert sys.getsizeof(v) == 48 - - def test_get_item(self) -> None: - v = append(vec[str](), '44') - assert v[0] == '44' - v = append(v, '56') - assert v[0] == '44' - assert v[1] == '56' - - v2 = vec[Optional[str]]([None, '12']) - assert v2[0] is None - assert v2[1] == '12' - - def test_get_item_error(self) -> None: - v = append(vec[str](), '1') - v = append(v, '2') - v = append(v, '3') - with self.assertRaises(IndexError): - v[3] - with self.assertRaises(IndexError): - v[-4] - n = 2 ** 100 - with self.assertRaises(IndexError): - v[n] - - def test_get_item_negative(self) -> None: - v = vec[str](['2', '5', '7']) - assert v[-1] == '7' - assert v[-2] == '5' - assert v[-3] == '2' - with self.assertRaises(IndexError): - v[-4] - with self.assertRaises(IndexError): - v[-2**62] - - def test_len(self) -> None: - v = vec[str]() +def assert_neq(x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + + +def test_vec_indexing() -> None: + assert vec[str] is not vec[i64] + assert type(vec[str]) is not type + assert type(vec[Optional[str]]) is type(vec[str]) + + +def test_type_str() -> None: + assert str(vec[str]) == "" + assert str(vec[Optional[str]]) == "" + + +def test_construct_empty() -> None: + v = vec[str]() + assert len(v) == 0 + v2 = vec[Optional[str]]() + assert len(v2) == 0 + assert type(v) is type(v2) + + +def test_construct_or_union() -> None: + if sys.version_info >= (3, 10): + v = vec[str | None]() assert len(v) == 0 - v = append(v, '1') - assert len(v) == 1 - v = append(v, '2') - assert len(v) == 2 - for i in range(100): - v = append(v, 'i') - assert len(v) == i + 3 - - def test_set_item(self) -> None: - v = vec[str]() - v = append(v, '1') - v = append(v, '2') - v[0] = '3' - assert v[0] == '3' - assert v[1] == '2' - v[1] = '5' - assert v[0] == '3' - assert v[1] == '5' - - def test_set_item_optional(self) -> None: - v = vec[Optional[str]]() - v = append(v, '1') + assert str(v) == 'vec[str | None]([])' + + +def test_construct_from_initializer() -> None: + v = vec[str](['xyz']) + assert len(v) == 1 + assert v[0] == 'xyz' + v = vec[str](x + '!' for x in ['x', '', 'foo', 'bar']) + assert len(v) == 4 + assert v[0] == 'x!' + assert v[1] == '!' + assert v[2] == 'foo!' + assert v[3] == 'bar!' + with assertRaises(TypeError): + vec[str](1) + with assertRaises(TypeError): + vec[str]([1]) + with assertRaises(TypeError): + vec[str]([None]) + + +def test_construct_from_initializer_optional() -> None: + v = vec[Optional[str]]([None, 'xyz']) + assert len(v) == 2 + assert v[0] is None + assert v[1] == 'xyz' + v = vec[Optional[str]](x for x in ['x', None, 'foo', 'bar']) + assert len(v) == 4 + assert v[0] == 'x' + assert v[1] is None + assert v[2] == 'foo' + assert v[3] == 'bar' + with assertRaises(TypeError): + vec[Optional[str]](1) + with assertRaises(TypeError): + vec[Optional[str]]([1]) + + +def test_append() -> None: + v = vec[str]() + v = append(v, '1') + assert str(v) == "vec[str](['1'])" + for n in range(2, 10): + v = append(v, str(n)) + assert str(v) == "vec[str](['1', '2', '3', '4', '5', '6', '7', '8', '9'])" + + v2 = vec[Optional[str]]() + v2 = append(v2, 'x') + v2 = append(v2, None) + assert str(v2) == "vec[str | None](['x', None])" + + +def test_append_error() -> None: + v: Any = vec[str]() + with assertRaises(TypeError): + v = append(v, b'x') + with assertRaises(TypeError): + v = append(v, ['x']) + with assertRaises(TypeError): v = append(v, None) - v[0] = '3' - assert v[0] == '3' - assert v[1] is None - v[1] = '5' - assert v[0] == '3' - assert v[1] == '5' - v[1] = None - assert v[0] == '3' - assert v[1] is None - - def test_set_item_error(self) -> None: - v = append(vec[str](), '1') - v = append(v, '2') - v = append(v, '3') - with self.assertRaises(IndexError): - v[3] = '1' - with self.assertRaises(TypeError): - v[0] = b'x' - with self.assertRaises(TypeError): - v[0] = vec[str]() - with self.assertRaises(TypeError): - v[0] = None - assert v[0] == '1' - - def test_set_item_error_optional(self) -> None: - v = append(vec[Optional[str]](), '1') - v = append(v, None) - v = append(v, '3') - with self.assertRaises(IndexError): - v[3] = '1' - with self.assertRaises(IndexError): - v[3] = None - with self.assertRaises(IndexError): - v[-4] = '1' - with self.assertRaises(IndexError): - v[-4] = None - with self.assertRaises(TypeError): - v[0] = b'x' - with self.assertRaises(TypeError): - v[0] = vec[str]() - with self.assertRaises(TypeError): - v[0] = vec[Optional[str]]() - assert v[0] == '1' - - def test_set_item_negative(self) -> None: - v = vec[str]() - v = append(v, '1') - v = append(v, '2') - v[-2] = '3' - assert v[0] == '3' - assert v[1] == '2' - v[-1] = '5' - assert v[0] == '3' - assert v[1] == '5' - with self.assertRaises(IndexError): - v[-3] = '1' - - def test_nested_in_type(self) -> None: - cast(List[vec[str]], []) - - def test_cannot_overload_isinstance(self) -> None: - class M(type): - def __instancecheck__(cls, instance): - return isinstance(instance, str) - - class C(metaclass=M): pass - - assert isinstance('', C) - v = vec[C]() - v = append(v, C()) - with self.assertRaises(TypeError): - v = append(v, '') - with self.assertRaises(TypeError): - v[0] = '' - - def test_add_subclass_instance(self) -> None: - class SS(str): - pass - - v = vec[str]() - v = append(v, SS()) - v[0] = SS() - - v2 = vec[SS]() - with self.assertRaises(TypeError): - v2 = append(v2, '') - - def test_equality(self) -> None: - self.assert_eq(vec[str](), vec[str]()) - self.assert_eq(vec[str](['x']), vec[str](['x'])) - v = vec[str](['x']) - self.assert_eq(v, v) - self.assert_eq(vec[str](['x', 'y']), vec[str](['x', 'y'])) - - self.assert_neq(vec[str](), vec[str](['x'])) - self.assert_neq(vec[str](['x', 'y']), vec[str](['x'])) - self.assert_neq(vec[str](['x']), vec[str](['y'])) - self.assert_neq(vec[str](['x', 'y']), vec[str](['y', 'y'])) - self.assert_neq(vec[str](['x', 'y']), vec[str](['x', 'x'])) - - self.assert_eq(vec[Optional[str]](), vec[Optional[str]]()) - self.assert_eq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', None])) - self.assert_neq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', 'y'])) - - def test_equality_different_types(self) -> None: - self.assert_neq(vec[str](['x']), ['x']) - self.assert_neq(vec[str](['x']), ('x',)) - self.assert_neq(vec[str](['x']), vec[object](['x'])) - self.assert_neq(vec[str](), vec[object]()) - self.assert_neq(vec[str](), vec[Optional[str]]()) - - def test_exception_from_eq(self) -> None: - class C: - def __eq__(self, other): - raise RuntimeError() - - with self.assertRaises(RuntimeError): - vec[C]([C()]) == vec[C]([C()]) - - def assert_eq(self, x: object, y: object) -> None: - assert x == y - assert y == x - b = x != y - assert not b - b = y != x - assert not b - - def assert_neq(self, x: object, y: object) -> None: - assert x != y - assert y != x - b = x == y - assert not b - b = y == x - assert not b - - def test_iteration(self) -> None: - for x in vec[str](): - assert False - a = [] - for x in vec[str](['x']): - a.append(x) - assert a == ['x'] - a = [] - for x in vec[str](['11', '23', '45', '64']): - a.append(x) - assert a == ['11', '23', '45', '64'] - a = [] - for x in vec[Optional[str]](['11', None, '45', '64']): - a.append(x) - assert a == ['11', None, '45', '64'] - - def test_slicing(self) -> None: - v = vec[str](str(i) for i in range(5)) - assert v[1:4] == vec[str](['1', '2', '3']) - assert v[2:-1] == vec[str](['2', '3']) - assert v[-2:-1] == vec[str](['3']) - assert v[1:] == vec[str](['1', '2', '3', '4']) - assert v[:-1] == vec[str](['0', '1', '2', '3']) - assert v[:] == v - assert v[:] is not v - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[str](['2', '3', '4']) - assert v[-100:2] == vec[str](['0', '1']) - assert v[5:100] == vec[str]([]) - assert v[50:100] == vec[str]([]) - assert v[-100:-50] == vec[str]([]) - - def test_slicing_optional(self) -> None: - v = vec[Optional[str]](['x', None, 'y']) - assert v[:2] == vec[Optional[str]](['x', None]) - - def test_slicing_with_step(self) -> None: - v = vec[str](str(i) for i in range(10)) - assert v[1:5:2] == vec[str](['1', '3']) - assert v[1:6:2] == vec[str](['1', '3', '5']) - assert v[5:1:-2] == vec[str](['5', '3']) - assert v[6:1:-2] == vec[str](['6', '4', '2']) - v = vec[str](str(i) for i in range(5)) - assert v[::-1] == vec[str](['4', '3', '2', '1', '0']) - with self.assertRaises(ValueError): - v[1:3:0] - - def test_contains(self) -> None: - v = vec[str](str(i) for i in range(2, 7)) - for i in range(2, 7): - assert str(i) in v - assert 'x' not in v - assert '22' not in v - vv = vec[str]() - assert '3' not in vv - - v2 = vec[Optional[str]]() - assert '3' not in v2 - assert None not in v2 - v2 = vec[Optional[str]](['3', None]) - assert '3' in v2 - assert None in v2 - assert '4' not in v2 - - def test_remove(self) -> None: - a = ['4x', '7x', '9x'] - for i in a: - v = vec[str](a) - v = remove(v, i) - assert v == vec[str]([j for j in a if j != i]) - v = vec[str](a) - # Make sure we have a different id - v = remove(v, (i + 'x')[:-1]) - assert v == vec[str]([j for j in a if j != i]) + assert len(v) == 0 + + v2: Any = vec[Optional[str]]() + with assertRaises(TypeError): + v2 = append(v2, b'x') + + +def test_repr() -> None: + assert str(vec[str]()) == "vec[str]([])" + assert str(vec[Optional[str]]([None, 'x'])) == "vec[str | None]([None, 'x'])" + + +def test_size() -> None: + v = vec[str]() + assert getsizeof(v) == 48 + v = append(v, '123') + assert getsizeof(v) == 48 + + +def test_get_item() -> None: + v = append(vec[str](), '44') + assert v[0] == '44' + v = append(v, '56') + assert v[0] == '44' + assert v[1] == '56' + + v2 = vec[Optional[str]]([None, '12']) + assert v2[0] is None + assert v2[1] == '12' + + +def test_get_item_error() -> None: + v = append(vec[str](), '1') + v = append(v, '2') + v = append(v, '3') + with assertRaises(IndexError): + v[3] + with assertRaises(IndexError): + v[-4] + n = 2 ** 100 + with assertRaises(IndexError): + v[n] + + +def test_get_item_negative() -> None: + v = vec[str](['2', '5', '7']) + assert v[-1] == '7' + assert v[-2] == '5' + assert v[-3] == '2' + with assertRaises(IndexError): + v[-4] + with assertRaises(IndexError): + v[-2**62] + + +def test_len() -> None: + v = vec[str]() + assert len(v) == 0 + v = append(v, '1') + assert len(v) == 1 + v = append(v, '2') + assert len(v) == 2 + for i in range(100): + v = append(v, 'i') + assert len(v) == i + 3 + + +def test_set_item() -> None: + v = vec[str]() + v = append(v, '1') + v = append(v, '2') + v[0] = '3' + assert v[0] == '3' + assert v[1] == '2' + v[1] = '5' + assert v[0] == '3' + assert v[1] == '5' + + +def test_set_item_optional() -> None: + v = vec[Optional[str]]() + v = append(v, '1') + v = append(v, None) + v[0] = '3' + assert v[0] == '3' + assert v[1] is None + v[1] = '5' + assert v[0] == '3' + assert v[1] == '5' + v[1] = None + assert v[0] == '3' + assert v[1] is None + + +def test_set_item_error() -> None: + v = append(vec[str](), '1') + v = append(v, '2') + v = append(v, '3') + with assertRaises(IndexError): + v[3] = '1' + with assertRaises(TypeError): + v[0] = b'x' + with assertRaises(TypeError): + v[0] = vec[str]() + with assertRaises(TypeError): + v[0] = None + assert v[0] == '1' + + +def test_set_item_error_optional() -> None: + v = append(vec[Optional[str]](), '1') + v = append(v, None) + v = append(v, '3') + with assertRaises(IndexError): + v[3] = '1' + with assertRaises(IndexError): + v[3] = None + with assertRaises(IndexError): + v[-4] = '1' + with assertRaises(IndexError): + v[-4] = None + with assertRaises(TypeError): + v[0] = b'x' + with assertRaises(TypeError): + v[0] = vec[str]() + with assertRaises(TypeError): + v[0] = vec[Optional[str]]() + assert v[0] == '1' + + +def test_set_item_negative() -> None: + v = vec[str]() + v = append(v, '1') + v = append(v, '2') + v[-2] = '3' + assert v[0] == '3' + assert v[1] == '2' + v[-1] = '5' + assert v[0] == '3' + assert v[1] == '5' + with assertRaises(IndexError): + v[-3] = '1' + + +def test_nested_in_type() -> None: + cast(List[vec[str]], []) + + +def test_cannot_overload_isinstance() -> None: + class M(type): + def __instancecheck__(cls, instance): + return isinstance(instance, str) + + class C(metaclass=M): pass + + assert isinstance('', C) + v = vec[C]() + v = append(v, C()) + with assertRaises(TypeError): + v = append(v, '') + with assertRaises(TypeError): + v[0] = '' + + +def test_add_subclass_instance() -> None: + class SS(str): + pass + + v = vec[str]() + v = append(v, SS()) + v[0] = SS() + + v2 = vec[SS]() + with assertRaises(TypeError): + v2 = append(v2, '') + + +def test_equality() -> None: + assert_eq(vec[str](), vec[str]()) + assert_eq(vec[str](['x']), vec[str](['x'])) + v = vec[str](['x']) + assert_eq(v, v) + assert_eq(vec[str](['x', 'y']), vec[str](['x', 'y'])) + + assert_neq(vec[str](), vec[str](['x'])) + assert_neq(vec[str](['x', 'y']), vec[str](['x'])) + assert_neq(vec[str](['x']), vec[str](['y'])) + assert_neq(vec[str](['x', 'y']), vec[str](['y', 'y'])) + assert_neq(vec[str](['x', 'y']), vec[str](['x', 'x'])) + + assert_eq(vec[Optional[str]](), vec[Optional[str]]()) + assert_eq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', None])) + assert_neq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', 'y'])) + + +def test_equality_different_types() -> None: + assert_neq(vec[str](['x']), ['x']) + assert_neq(vec[str](['x']), ('x',)) + assert_neq(vec[str](['x']), vec[object](['x'])) + assert_neq(vec[str](), vec[object]()) + assert_neq(vec[str](), vec[Optional[str]]()) + + +def test_exception_from_eq() -> None: + class C: + def __eq__(self, other): + raise RuntimeError() + + with assertRaises(RuntimeError): + vec[C]([C()]) == vec[C]([C()]) + + +def test_iteration() -> None: + for x in vec[str](): + assert False + a = [] + for x in vec[str](['x']): + a.append(x) + assert a == ['x'] + a = [] + for x in vec[str](['11', '23', '45', '64']): + a.append(x) + assert a == ['11', '23', '45', '64'] + a = [] + for x in vec[Optional[str]](['11', None, '45', '64']): + a.append(x) + assert a == ['11', None, '45', '64'] + + +def test_slicing() -> None: + v = vec[str](str(i) for i in range(5)) + assert v[1:4] == vec[str](['1', '2', '3']) + assert v[2:-1] == vec[str](['2', '3']) + assert v[-2:-1] == vec[str](['3']) + assert v[1:] == vec[str](['1', '2', '3', '4']) + assert v[:-1] == vec[str](['0', '1', '2', '3']) + assert v[:] == v + assert v[:] is not v + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[str](['2', '3', '4']) + assert v[-100:2] == vec[str](['0', '1']) + assert v[5:100] == vec[str]([]) + assert v[50:100] == vec[str]([]) + assert v[-100:-50] == vec[str]([]) + + +def test_slicing_optional() -> None: + v = vec[Optional[str]](['x', None, 'y']) + assert v[:2] == vec[Optional[str]](['x', None]) + + +def test_slicing_with_step() -> None: + v = vec[str](str(i) for i in range(10)) + assert v[1:5:2] == vec[str](['1', '3']) + assert v[1:6:2] == vec[str](['1', '3', '5']) + assert v[5:1:-2] == vec[str](['5', '3']) + assert v[6:1:-2] == vec[str](['6', '4', '2']) + v = vec[str](str(i) for i in range(5)) + assert v[::-1] == vec[str](['4', '3', '2', '1', '0']) + with assertRaises(ValueError): + v[1:3:0] + + +def test_contains() -> None: + v = vec[str](str(i) for i in range(2, 7)) + for i in range(2, 7): + assert str(i) in v + assert 'x' not in v + assert '22' not in v + vv = vec[str]() + assert '3' not in vv + + v2 = vec[Optional[str]]() + assert '3' not in v2 + assert None not in v2 + v2 = vec[Optional[str]](['3', None]) + assert '3' in v2 + assert None in v2 + assert '4' not in v2 + + +def test_remove() -> None: + a = ['4x', '7x', '9x'] + for i in a: v = vec[str](a) - v = remove(v, '4x') - v = remove(v, '7x') - v = remove(v, '9x') - assert v == vec[str]() - with self.assertRaises(ValueError): - remove(v, '4') - v = append(v, '5') - with self.assertRaises(ValueError): - remove(v, '7') - v = remove(v, '5') - assert len(v) == 0 - v = vec[str](['x', 'x', 'x']) - v = remove(v, 'x') - assert v == vec[str](['x', 'x']) - v = remove(v, 'x') - assert v == vec[str](['x']) - v = remove(v, 'x') - assert v == vec[str]() - b: Any = b'x' - with self.assertRaises(TypeError): - remove(v, b) - - def test_remove_optional(self) -> None: - v = vec[Optional[str]](['x', None]) - v = remove(v, 'x') - assert str(v) == "vec[str | None]([None])" - v = remove(v, None) - assert str(v) == "vec[str | None]([])" - - def test_pop_last(self) -> None: - v = vec[str](['4', '7', '9']) - v, item = pop(v) - assert item == '9' - assert v == vec[str](['4', '7']) - v, item = pop(v) - assert item == '7' - assert v == vec[str](['4']) - v, item = pop(v) - assert item == '4' - assert v == vec[str]() - with self.assertRaises(IndexError): - pop(v) - - def test_pop_index(self) -> None: - v = vec[str](['4', '7', '9', '15', '22']) - v, item = pop(v, 0) - assert item == '4' - assert v == vec[str](['7', '9', '15', '22']) - v, item = pop(v, -1) - assert item == '22' - assert v == vec[str](['7', '9', '15']) - v, item = pop(v, 1) - assert item == '9' - assert v == vec[str](['7', '15']) - - with self.assertRaises(IndexError): - pop(v, 2) - - with self.assertRaises(IndexError): - pop(v, -3) - - v, item = pop(v, -2) - assert item == '7' - assert v == vec[str](['15']) - v, item = pop(v, 0) - assert item == '15' - assert v == vec[str]() + v = remove(v, i) + assert v == vec[str]([j for j in a if j != i]) + v = vec[str](a) + # Make sure we have a different id + v = remove(v, (i + 'x')[:-1]) + assert v == vec[str]([j for j in a if j != i]) + v = vec[str](a) + v = remove(v, '4x') + v = remove(v, '7x') + v = remove(v, '9x') + assert v == vec[str]() + with assertRaises(ValueError): + remove(v, '4') + v = append(v, '5') + with assertRaises(ValueError): + remove(v, '7') + v = remove(v, '5') + assert len(v) == 0 + v = vec[str](['x', 'x', 'x']) + v = remove(v, 'x') + assert v == vec[str](['x', 'x']) + v = remove(v, 'x') + assert v == vec[str](['x']) + v = remove(v, 'x') + assert v == vec[str]() + b: Any = b'x' + with assertRaises(TypeError): + remove(v, b) + + +def test_remove_optional() -> None: + v = vec[Optional[str]](['x', None]) + v = remove(v, 'x') + assert str(v) == "vec[str | None]([None])" + v = remove(v, None) + assert str(v) == "vec[str | None]([])" + + +def test_pop_last() -> None: + v = vec[str](['4', '7', '9']) + v, item = pop(v) + assert item == '9' + assert v == vec[str](['4', '7']) + v, item = pop(v) + assert item == '7' + assert v == vec[str](['4']) + v, item = pop(v) + assert item == '4' + assert v == vec[str]() + with assertRaises(IndexError): + pop(v) + + +def test_pop_index() -> None: + v = vec[str](['4', '7', '9', '15', '22']) + v, item = pop(v, 0) + assert item == '4' + assert v == vec[str](['7', '9', '15', '22']) + v, item = pop(v, -1) + assert item == '22' + assert v == vec[str](['7', '9', '15']) + v, item = pop(v, 1) + assert item == '9' + assert v == vec[str](['7', '15']) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, item = pop(v, -2) + assert item == '7' + assert v == vec[str](['15']) + v, item = pop(v, 0) + assert item == '15' + assert v == vec[str]() From 065b1448c4ebd500b68597ba7ccee4b6f52c6fe5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:36:57 +0000 Subject: [PATCH 161/208] Fix spacing --- mypyc/test-data/run-vecs-t-interp.test | 36 -------------------------- 1 file changed, 36 deletions(-) diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test index e17a2e2d2e926..2630f393dc2ff 100644 --- a/mypyc/test-data/run-vecs-t-interp.test +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -15,7 +15,6 @@ pop: Any = librt.vecs.pop getsizeof: Any = getattr(sys, "getsizeof") i64: Any = mypy_extensions.i64 - def assert_eq(x: object, y: object) -> None: assert x == y assert y == x @@ -24,7 +23,6 @@ def assert_eq(x: object, y: object) -> None: b = y != x assert not b - def assert_neq(x: object, y: object) -> None: assert x != y assert y != x @@ -33,18 +31,15 @@ def assert_neq(x: object, y: object) -> None: b = y == x assert not b - def test_vec_indexing() -> None: assert vec[str] is not vec[i64] assert type(vec[str]) is not type assert type(vec[Optional[str]]) is type(vec[str]) - def test_type_str() -> None: assert str(vec[str]) == "" assert str(vec[Optional[str]]) == "" - def test_construct_empty() -> None: v = vec[str]() assert len(v) == 0 @@ -52,14 +47,12 @@ def test_construct_empty() -> None: assert len(v2) == 0 assert type(v) is type(v2) - def test_construct_or_union() -> None: if sys.version_info >= (3, 10): v = vec[str | None]() assert len(v) == 0 assert str(v) == 'vec[str | None]([])' - def test_construct_from_initializer() -> None: v = vec[str](['xyz']) assert len(v) == 1 @@ -77,7 +70,6 @@ def test_construct_from_initializer() -> None: with assertRaises(TypeError): vec[str]([None]) - def test_construct_from_initializer_optional() -> None: v = vec[Optional[str]]([None, 'xyz']) assert len(v) == 2 @@ -94,7 +86,6 @@ def test_construct_from_initializer_optional() -> None: with assertRaises(TypeError): vec[Optional[str]]([1]) - def test_append() -> None: v = vec[str]() v = append(v, '1') @@ -108,7 +99,6 @@ def test_append() -> None: v2 = append(v2, None) assert str(v2) == "vec[str | None](['x', None])" - def test_append_error() -> None: v: Any = vec[str]() with assertRaises(TypeError): @@ -123,19 +113,16 @@ def test_append_error() -> None: with assertRaises(TypeError): v2 = append(v2, b'x') - def test_repr() -> None: assert str(vec[str]()) == "vec[str]([])" assert str(vec[Optional[str]]([None, 'x'])) == "vec[str | None]([None, 'x'])" - def test_size() -> None: v = vec[str]() assert getsizeof(v) == 48 v = append(v, '123') assert getsizeof(v) == 48 - def test_get_item() -> None: v = append(vec[str](), '44') assert v[0] == '44' @@ -147,7 +134,6 @@ def test_get_item() -> None: assert v2[0] is None assert v2[1] == '12' - def test_get_item_error() -> None: v = append(vec[str](), '1') v = append(v, '2') @@ -160,7 +146,6 @@ def test_get_item_error() -> None: with assertRaises(IndexError): v[n] - def test_get_item_negative() -> None: v = vec[str](['2', '5', '7']) assert v[-1] == '7' @@ -171,7 +156,6 @@ def test_get_item_negative() -> None: with assertRaises(IndexError): v[-2**62] - def test_len() -> None: v = vec[str]() assert len(v) == 0 @@ -183,7 +167,6 @@ def test_len() -> None: v = append(v, 'i') assert len(v) == i + 3 - def test_set_item() -> None: v = vec[str]() v = append(v, '1') @@ -195,7 +178,6 @@ def test_set_item() -> None: assert v[0] == '3' assert v[1] == '5' - def test_set_item_optional() -> None: v = vec[Optional[str]]() v = append(v, '1') @@ -210,7 +192,6 @@ def test_set_item_optional() -> None: assert v[0] == '3' assert v[1] is None - def test_set_item_error() -> None: v = append(vec[str](), '1') v = append(v, '2') @@ -225,7 +206,6 @@ def test_set_item_error() -> None: v[0] = None assert v[0] == '1' - def test_set_item_error_optional() -> None: v = append(vec[Optional[str]](), '1') v = append(v, None) @@ -246,7 +226,6 @@ def test_set_item_error_optional() -> None: v[0] = vec[Optional[str]]() assert v[0] == '1' - def test_set_item_negative() -> None: v = vec[str]() v = append(v, '1') @@ -260,11 +239,9 @@ def test_set_item_negative() -> None: with assertRaises(IndexError): v[-3] = '1' - def test_nested_in_type() -> None: cast(List[vec[str]], []) - def test_cannot_overload_isinstance() -> None: class M(type): def __instancecheck__(cls, instance): @@ -280,7 +257,6 @@ def test_cannot_overload_isinstance() -> None: with assertRaises(TypeError): v[0] = '' - def test_add_subclass_instance() -> None: class SS(str): pass @@ -293,7 +269,6 @@ def test_add_subclass_instance() -> None: with assertRaises(TypeError): v2 = append(v2, '') - def test_equality() -> None: assert_eq(vec[str](), vec[str]()) assert_eq(vec[str](['x']), vec[str](['x'])) @@ -311,7 +286,6 @@ def test_equality() -> None: assert_eq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', None])) assert_neq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', 'y'])) - def test_equality_different_types() -> None: assert_neq(vec[str](['x']), ['x']) assert_neq(vec[str](['x']), ('x',)) @@ -319,7 +293,6 @@ def test_equality_different_types() -> None: assert_neq(vec[str](), vec[object]()) assert_neq(vec[str](), vec[Optional[str]]()) - def test_exception_from_eq() -> None: class C: def __eq__(self, other): @@ -328,7 +301,6 @@ def test_exception_from_eq() -> None: with assertRaises(RuntimeError): vec[C]([C()]) == vec[C]([C()]) - def test_iteration() -> None: for x in vec[str](): assert False @@ -345,7 +317,6 @@ def test_iteration() -> None: a.append(x) assert a == ['11', None, '45', '64'] - def test_slicing() -> None: v = vec[str](str(i) for i in range(5)) assert v[1:4] == vec[str](['1', '2', '3']) @@ -363,12 +334,10 @@ def test_slicing() -> None: assert v[50:100] == vec[str]([]) assert v[-100:-50] == vec[str]([]) - def test_slicing_optional() -> None: v = vec[Optional[str]](['x', None, 'y']) assert v[:2] == vec[Optional[str]](['x', None]) - def test_slicing_with_step() -> None: v = vec[str](str(i) for i in range(10)) assert v[1:5:2] == vec[str](['1', '3']) @@ -380,7 +349,6 @@ def test_slicing_with_step() -> None: with assertRaises(ValueError): v[1:3:0] - def test_contains() -> None: v = vec[str](str(i) for i in range(2, 7)) for i in range(2, 7): @@ -398,7 +366,6 @@ def test_contains() -> None: assert None in v2 assert '4' not in v2 - def test_remove() -> None: a = ['4x', '7x', '9x'] for i in a: @@ -432,7 +399,6 @@ def test_remove() -> None: with assertRaises(TypeError): remove(v, b) - def test_remove_optional() -> None: v = vec[Optional[str]](['x', None]) v = remove(v, 'x') @@ -440,7 +406,6 @@ def test_remove_optional() -> None: v = remove(v, None) assert str(v) == "vec[str | None]([])" - def test_pop_last() -> None: v = vec[str](['4', '7', '9']) v, item = pop(v) @@ -455,7 +420,6 @@ def test_pop_last() -> None: with assertRaises(IndexError): pop(v) - def test_pop_index() -> None: v = vec[str](['4', '7', '9', '15', '22']) v, item = pop(v, 0) From 59fdf8ce605196ee3ace967cbae72e4f17c5374c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:42:22 +0000 Subject: [PATCH 162/208] WIP fixing tests --- mypyc/test-data/run-vecs-t-interp.test | 9 ++++++--- mypyc/test/test_run.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test index 2630f393dc2ff..a3376ddd72321 100644 --- a/mypyc/test-data/run-vecs-t-interp.test +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -1,7 +1,8 @@ -[case testLibrtVecsI64Interpreted_librt_experimental] +[case testLibrtVecsTInterpreted_librt_experimental] # Test cases for vec[], using generic operations (simulates use from interpreted code). import sys -from typing import cast, List, Optional, Any +from typing import cast, List, Any +import typing import librt.vecs import mypy_extensions @@ -14,6 +15,8 @@ remove: Any = librt.vecs.remove pop: Any = librt.vecs.pop getsizeof: Any = getattr(sys, "getsizeof") i64: Any = mypy_extensions.i64 +Optional: Any = typing.Optional +version_info: Any = sys.version_info def assert_eq(x: object, y: object) -> None: assert x == y @@ -48,7 +51,7 @@ def test_construct_empty() -> None: assert type(v) is type(v2) def test_construct_or_union() -> None: - if sys.version_info >= (3, 10): + if version_info >= (3, 10): v = vec[str | None]() assert len(v) == 0 assert str(v) == 'vec[str | None]([])' diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index cd31f718af097..d9f5141b72858 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -79,6 +79,7 @@ "run-base64.test", "run-match.test", "run-vecs-i64-interp.test", + "run-vecs-t-interp.test", ] if sys.version_info >= (3, 12): From ba069eec4e0cdd24d1449f4ce0f0b1b77235f8a6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 14:51:26 +0000 Subject: [PATCH 163/208] Fix tests --- mypyc/test-data/run-vecs-t-interp.test | 50 +++++++++++++++----------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test index a3376ddd72321..8b656fcf0d06d 100644 --- a/mypyc/test-data/run-vecs-t-interp.test +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -8,6 +8,7 @@ import librt.vecs import mypy_extensions from testutil import assertRaises +from helpers import ClassWithCustomMeta vec: Any = librt.vecs.vec append: Any = librt.vecs.append @@ -15,9 +16,11 @@ remove: Any = librt.vecs.remove pop: Any = librt.vecs.pop getsizeof: Any = getattr(sys, "getsizeof") i64: Any = mypy_extensions.i64 -Optional: Any = typing.Optional version_info: Any = sys.version_info +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + def assert_eq(x: object, y: object) -> None: assert x == y assert y == x @@ -34,6 +37,14 @@ def assert_neq(x: object, y: object) -> None: b = y == x assert not b +@mypy_extensions.mypyc_attr(native_class=False) +class StringSubclass(str): + pass + +class ClassWithRaisingEq: + def __eq__(self, other): + raise RuntimeError() + def test_vec_indexing() -> None: assert vec[str] is not vec[i64] assert type(vec[str]) is not type @@ -246,29 +257,20 @@ def test_nested_in_type() -> None: cast(List[vec[str]], []) def test_cannot_overload_isinstance() -> None: - class M(type): - def __instancecheck__(cls, instance): - return isinstance(instance, str) - - class C(metaclass=M): pass - - assert isinstance('', C) - v = vec[C]() - v = append(v, C()) + assert isinstance('', ClassWithCustomMeta) + v = vec[ClassWithCustomMeta]() + v = append(v, ClassWithCustomMeta()) with assertRaises(TypeError): v = append(v, '') with assertRaises(TypeError): v[0] = '' def test_add_subclass_instance() -> None: - class SS(str): - pass - v = vec[str]() - v = append(v, SS()) - v[0] = SS() + v = append(v, StringSubclass()) + v[0] = StringSubclass() - v2 = vec[SS]() + v2 = vec[StringSubclass]() with assertRaises(TypeError): v2 = append(v2, '') @@ -297,12 +299,8 @@ def test_equality_different_types() -> None: assert_neq(vec[str](), vec[Optional[str]]()) def test_exception_from_eq() -> None: - class C: - def __eq__(self, other): - raise RuntimeError() - with assertRaises(RuntimeError): - vec[C]([C()]) == vec[C]([C()]) + vec[ClassWithRaisingEq]([ClassWithRaisingEq()]) == vec[ClassWithRaisingEq]([ClassWithRaisingEq()]) def test_iteration() -> None: for x in vec[str](): @@ -447,3 +445,13 @@ def test_pop_index() -> None: v, item = pop(v, 0) assert item == '15' assert v == vec[str]() + +[file helpers.py] +# Test helper classes + +class MetaWithInstanceCheck(type): + def __instancecheck__(cls, instance): + return isinstance(instance, str) + +class ClassWithCustomMeta(metaclass=MetaWithInstanceCheck): + pass From bad2b11e58a1207643a2bdb9ba0c9ee62a0c1af6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 15:04:26 +0000 Subject: [PATCH 164/208] Fix reference counting --- mypyc/lib-rt/vecs/vec_t.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index bab74373c51bb..203510267d6b2 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -181,12 +181,16 @@ static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { if (!VecT_ItemCheck(v, o, v.buf->item_type)) return -1; if ((size_t)i < (size_t)v.len) { + PyObject *old = v.buf->items[i]; Py_INCREF(o); v.buf->items[i] = o; + Py_XDECREF(old); return 0; } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { + PyObject *old = v.buf->items[i + v.len]; Py_INCREF(o); v.buf->items[i + v.len] = o; + Py_XDECREF(old); return 0; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); From 12ce9c50abb3415683f4cc8e003b8a9094407f1a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 17:32:39 +0000 Subject: [PATCH 165/208] Minor refactoring --- mypyc/lib-rt/vecs/vec_t.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 203510267d6b2..55648ae252450 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -141,13 +141,13 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { if (i == -1 && PyErr_Occurred()) return NULL; if ((size_t)i < (size_t)vec.len) { - PyObject *item = vec.buf->items[i]; - Py_INCREF(item); - return item; + PyObject *result = vec.buf->items[i]; + Py_INCREF(result); + return result; } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { - PyObject *item = vec.buf->items[i + vec.len]; - Py_INCREF(item); - return item; + PyObject *result = vec.buf->items[i + vec.len]; + Py_INCREF(result); + return result; } else { PyErr_SetString(PyExc_IndexError, "index out of range"); return NULL; From eccef8df08794f4193f102ff561a0ca4530171ee Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 17:37:11 +0000 Subject: [PATCH 166/208] Work around mypy bug --- mypyc/test-data/run-vecs-t-interp.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test index 8b656fcf0d06d..eae5898be861c 100644 --- a/mypyc/test-data/run-vecs-t-interp.test +++ b/mypyc/test-data/run-vecs-t-interp.test @@ -198,7 +198,7 @@ def test_set_item_optional() -> None: v = append(v, None) v[0] = '3' assert v[0] == '3' - assert v[1] is None + assert v[1 + int()] is None # TODO: The '+ int()' is due to mypy type narrowing bug v[1] = '5' assert v[0] == '3' assert v[1] == '5' From 66b8ef436cf444c3d2ec3ecfa982f9d586555d05 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Thu, 22 Jan 2026 17:57:17 +0000 Subject: [PATCH 167/208] Improve error handling and add comments --- mypyc/lib-rt/vecs/vec_nested.c | 1 + mypyc/lib-rt/vecs/vec_t.c | 22 +++++++++++++++++----- mypyc/lib-rt/vecs/vec_template.c | 1 + 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 2557c0c892062..4397acd8056e7 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -33,6 +33,7 @@ static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { return res; } +// Box a nested vec value, stealing the buffer reference. PyObject *VecVec_Box(VecNested vec) { VecNestedObject *obj = PyObject_GC_New(VecNestedObject, &VecNestedType); if (obj == NULL) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 55648ae252450..22e14386a3c92 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -41,6 +41,7 @@ static VecT vec_alloc(Py_ssize_t size, size_t item_type) { return (VecT) { .buf = buf }; } +// Box a VecT value, stealing the buffer reference. PyObject *VecT_Box(VecT vec, size_t item_type) { // An unboxed empty vec may have a NULL buf, but a boxed vec must have it // allocated, since it contains the item type @@ -50,8 +51,10 @@ PyObject *VecT_Box(VecT vec, size_t item_type) { return NULL; } VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); - if (obj == NULL) + if (obj == NULL) { + Py_DECREF(vec.buf); return NULL; + } obj->vec = vec; PyObject_GC_Track(obj); return (PyObject *)obj; @@ -168,7 +171,11 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return VecT_Box(res, vec.buf->item_type); + PyObject *result = VecT_Box(res, vec.buf->item_type); + if (result == NULL) { + VEC_DECREF(res); + } + return result; } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -239,8 +246,10 @@ VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { Py_ssize_t new_size = 2 * cap + 1; // TODO: Avoid initializing to zero here VecT new = vec_alloc(new_size, vec.buf->item_type); - if (VEC_IS_ERROR(new)) + if (VEC_IS_ERROR(new)) { + Py_DECREF(x); return new; + } // Copy items to new vec. memcpy(new.buf->items, vec.buf->items, sizeof(PyObject *) * vec.len); memset(new.buf->items + vec.len, 0, sizeof(PyObject *) * (new_size - vec.len)); @@ -271,6 +280,8 @@ VecT VecT_Remove(VecT v, PyObject *arg) { for (; i < v.len - 1; i++) { items[i] = items[i + 1]; } + // Keep a duplicate item, since there could be another reference + // to the buffer with a longer length, and they expect a valid reference. Py_XINCREF(items[v.len - 1]); } v.len--; @@ -299,8 +310,9 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { result.f1 = items[index]; for (Py_ssize_t i = index; i < v.len - 1; i++) items[i] = items[i + 1]; - if (v.len > 0) - Py_XINCREF(items[v.len - 1]); + // Keep duplicate item, since there could be another reference + // to the buffer with a longer length, and they expect a valid reference. + Py_XINCREF(items[v.len - 1]); v.len--; VEC_INCREF(v); result.f0 = v; diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index f9f961380a647..28911d032b908 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -41,6 +41,7 @@ static void vec_dealloc(VEC_OBJECT *self) { PyObject_Del(self); } +// Box a vec[] value, stealing the buffer reference. PyObject *FUNC(Box)(VEC vec) { VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); if (obj == NULL) From f4f5d496da620ffa10eafc70533d38791115d094 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:10:20 +0000 Subject: [PATCH 168/208] Reference counting fix + update comments --- mypyc/lib-rt/vecs/vec_nested.c | 6 ++++-- mypyc/lib-rt/vecs/vec_t.c | 3 ++- mypyc/lib-rt/vecs/vec_template.c | 6 ++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 4397acd8056e7..2f7b5cf3cc863 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -33,11 +33,13 @@ static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { return res; } -// Box a nested vec value, stealing the buffer reference. +// Box a nested vec value, stealing 'vec'. On error, decref 'vec'. PyObject *VecVec_Box(VecNested vec) { VecNestedObject *obj = PyObject_GC_New(VecNestedObject, &VecNestedType); - if (obj == NULL) + if (obj == NULL) { + VEC_DECREF(vec); return NULL; + } obj->vec = vec; PyObject_GC_Track(obj); return (PyObject *)obj; diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 22e14386a3c92..bde4956dbdaa5 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -41,7 +41,7 @@ static VecT vec_alloc(Py_ssize_t size, size_t item_type) { return (VecT) { .buf = buf }; } -// Box a VecT value, stealing the buffer reference. +// Box a VecT value, stealing 'vec'. On failure, return NULL and decref 'vec'. PyObject *VecT_Box(VecT vec, size_t item_type) { // An unboxed empty vec may have a NULL buf, but a boxed vec must have it // allocated, since it contains the item type @@ -52,6 +52,7 @@ PyObject *VecT_Box(VecT vec, size_t item_type) { } VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); if (obj == NULL) { + // vec.buf is always defined, so no need for a NULL heck Py_DECREF(vec.buf); return NULL; } diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 28911d032b908..aca06b53d0e92 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -41,11 +41,13 @@ static void vec_dealloc(VEC_OBJECT *self) { PyObject_Del(self); } -// Box a vec[] value, stealing the buffer reference. +// Box a vec[] value, stealing 'vec'. On error, decref 'vec'. PyObject *FUNC(Box)(VEC vec) { VEC_OBJECT *obj = PyObject_New(VEC_OBJECT, &VEC_TYPE); - if (obj == NULL) + if (obj == NULL) { + VEC_DECREF(vec); return NULL; + } obj->vec = vec; return (PyObject *)obj; } From 953d0add9de7ae6337a13f9532e30a465afcca53 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:25:27 +0000 Subject: [PATCH 169/208] Fixes to error handling for vec[i64] etc. --- mypyc/lib-rt/vecs/vec_template.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index aca06b53d0e92..8a33e67c50978 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -255,6 +255,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } +// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { if (vec.buf && vec.len < VEC_CAP(vec)) { vec.buf->items[vec.len] = x; @@ -264,8 +265,12 @@ VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { Py_ssize_t cap = vec.buf ? VEC_CAP(vec) : 0; Py_ssize_t new_size = 2 * cap + 1; VEC new = vec_alloc(new_size); - if (VEC_IS_ERROR(new)) + if (VEC_IS_ERROR(new)) { + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(vec); return vec_error(); + } new.len = vec.len + 1; if (vec.len > 0) memcpy(new.buf->items, vec.buf->items, sizeof(ITEM_C_TYPE) * vec.len); @@ -275,6 +280,7 @@ VEC FUNC(Append)(VEC vec, ITEM_C_TYPE x) { } } +// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { for (Py_ssize_t i = 0; i < v.len; i++) { if (v.buf->items[i] == x) { @@ -287,9 +293,13 @@ VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); return vec_error(); } +// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { NAME(PopResult) result; @@ -298,6 +308,9 @@ NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); result.f0 = vec_error(); result.f1 = 0; return result; From 432a556d92af1a9b9d7e33caba6b01da9a17a724 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:27:58 +0000 Subject: [PATCH 170/208] Improve error handling for vec[t] --- mypyc/lib-rt/vecs/vec_t.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index bde4956dbdaa5..f31a9dfacfd3c 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -227,6 +227,7 @@ static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } +// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { if (vec.buf == NULL) { VecT new = vec_alloc(1, item_type); @@ -249,6 +250,9 @@ VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { VecT new = vec_alloc(new_size, vec.buf->item_type); if (VEC_IS_ERROR(new)) { Py_DECREF(x); + // The input vec is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(vec); return new; } // Copy items to new vec. @@ -263,6 +267,7 @@ VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { } } +// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. VecT VecT_Remove(VecT v, PyObject *arg) { PyObject **items = v.buf->items; for (Py_ssize_t i = 0; i < v.len; i++) { @@ -271,8 +276,12 @@ VecT VecT_Remove(VecT v, PyObject *arg) { match = 1; else { int itemcmp = PyObject_RichCompareBool(items[i], arg, Py_EQ); - if (itemcmp < 0) + if (itemcmp < 0) { + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); return vec_error(); + } match = itemcmp; } if (match) { @@ -291,9 +300,13 @@ VecT VecT_Remove(VecT v, PyObject *arg) { } } PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); return vec_error(); } +// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { VecTPopResult result; @@ -302,6 +315,9 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); result.f0 = vec_error(); result.f1 = NULL; return result; From e9675468ed988f34d5941be98582a0c692a966c1 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:29:27 +0000 Subject: [PATCH 171/208] Remove some TODO comments The called function steal the reference, so they'll be responsible for decrefing on failure. --- mypyc/lib-rt/vecs/librt_vecs.c | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 96a6b6a1463a4..7d53f91b509f9 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -481,7 +481,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI64_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI64_Box(v); } else if (VecU8_Check(vec)) { uint8_t x = VecU8_UnboxItem(item); @@ -492,7 +492,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecU8_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecU8_Box(v); } else if (VecFloat_Check(vec)) { double x = VecFloat_UnboxItem(item); @@ -503,7 +503,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecFloat_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecFloat_Box(v); } else if (VecI32_Check(vec)) { int32_t x = VecI32_UnboxItem(item); @@ -514,7 +514,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI32_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI32_Box(v); } else if (VecI16_Check(vec)) { int16_t x = VecI16_UnboxItem(item); @@ -525,7 +525,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI16_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI16_Box(v); } else if (VecBool_Check(vec)) { double x = VecBool_UnboxItem(item); @@ -536,7 +536,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecBool_Append(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecBool_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; @@ -556,7 +556,7 @@ static PyObject *vec_append(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecVec_Append(v, vecitem); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecVec_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", @@ -582,7 +582,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI64_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI64_Box(v); } else if (VecU8_Check(vec)) { uint8_t x = VecU8_UnboxItem(item); @@ -593,7 +593,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecU8_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecU8_Box(v); } else if (VecFloat_Check(vec)) { double x = VecFloat_UnboxItem(item); @@ -604,7 +604,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecFloat_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecFloat_Box(v); } else if (VecI32_Check(vec)) { int32_t x = VecI32_UnboxItem(item); @@ -615,7 +615,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI32_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI32_Box(v); } else if (VecI16_Check(vec)) { int16_t x = VecI16_UnboxItem(item); @@ -626,7 +626,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecI16_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecI16_Box(v); } else if (VecBool_Check(vec)) { char x = VecBool_UnboxItem(item); @@ -637,7 +637,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecBool_Remove(v, x); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecBool_Box(v); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; @@ -657,7 +657,7 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) VEC_INCREF(v); v = VecVec_Remove(v, vecitem); if (VEC_IS_ERROR(v)) - return NULL; // TODO: decref? + return NULL; return VecVec_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", From d8b6a7ce3dfa3e60a01449ab27394f31afd1bcfd Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:48:49 +0000 Subject: [PATCH 172/208] Improve nested vec error handling --- mypyc/lib-rt/vecs/vec_nested.c | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 2f7b5cf3cc863..812deaa63515f 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -220,7 +220,7 @@ PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { return res; } -// Steals reference to vec (but not x) +// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); @@ -234,6 +234,9 @@ VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { VecNested new = vec_alloc(new_size, vec.buf->item_type, vec.buf->depth); if (VEC_IS_ERROR(new)) { VEC_DECREF(x); + // The input vec is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(vec); return new; } // Copy items to new vec. @@ -249,12 +252,17 @@ VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { } } +// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { VecNestedBufItem *items = self.buf->items; PyObject *boxed_arg = VecVec_BoxItem(self, arg); - if (boxed_arg == NULL) + if (boxed_arg == NULL) { + // The input self is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(self); return vec_error(); + } for (Py_ssize_t i = 0; i < self.len; i++) { int match = 0; @@ -264,12 +272,18 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { PyObject *item = box_vec_item_by_index(self, i); if (item == NULL) { Py_DECREF(boxed_arg); + // The input self is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(self); return vec_error(); } int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); Py_DECREF(item); if (itemcmp < 0) { Py_DECREF(boxed_arg); + // The input self is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(self); return vec_error(); } match = itemcmp; @@ -290,9 +304,13 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { } Py_DECREF(boxed_arg); PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); + // The input self is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(self); return vec_error(); } +// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { VecNestedPopResult result; @@ -301,6 +319,9 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { if (index < 0 || index >= v.len) { PyErr_SetString(PyExc_IndexError, "index out of range"); + // The input v is being consumed/stolen by this function, so on error + // we must decref it to avoid leaking the buffer. + VEC_DECREF(v); result.f0 = vec_error(); result.f1.len = 0; result.f1.buf = NULL; From e3b156a9cd702a593e17cf6be946884a07d8896e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:50:43 +0000 Subject: [PATCH 173/208] WIP nested vec interpreted tests --- mypyc/test-data/run-vecs-nested-interp.test | 307 ++++++++++++++++++++ mypyc/test/test_run.py | 1 + 2 files changed, 308 insertions(+) create mode 100644 mypyc/test-data/run-vecs-nested-interp.test diff --git a/mypyc/test-data/run-vecs-nested-interp.test b/mypyc/test-data/run-vecs-nested-interp.test new file mode 100644 index 0000000000000..c4e8f4f3e2e00 --- /dev/null +++ b/mypyc/test-data/run-vecs-nested-interp.test @@ -0,0 +1,307 @@ +[case testLibrtVecsNestedInterpreted_librt_experimental] +# Test cases for nested vecs, using generic operations (simulates use from interpreted code). + +import sys +from typing import Optional + +from vecs import vec, append, remove, pop + +from mypy_extensions import i64 + + +class TestVecT(unittest.TestCase): + def test_vec_indexing(self) -> None: + assert type(vec[vec[str]]) is type(vec[str]) + assert type(vec[vec[Optional[str]]]) is type(vec[str]) + assert type(vec[vec[i64]]) is type(vec[str]) + assert type(vec[vec[vec[vec[vec[vec[str]]]]]]) is type(vec[str]) + with self.assertRaises(TypeError): + vec[Optional[vec[str]]] + with self.assertRaises(TypeError): + vec[Optional[vec[Optional[str]]]] + + def test_type_str(self) -> None: + assert str(vec[Optional[str]]) == "" + assert str(vec[vec[str]]) == "" + assert str(vec[vec[i64]]) == "" + assert str(vec[vec[vec[vec[vec[str]]]]]) == "" + assert str( + vec[vec[vec[Optional[str]]]]) == "" + + def test_construct_empty_nested(self) -> None: + v = vec[vec[str]]() + assert len(v) == 0 + v2 = vec[vec[vec[vec[vec[str]]]]]() + assert len(v2) == 0 + v3 = vec[vec[i64]]() + assert len(v3) == 0 + + def test_construct_empty_nested_optional(self) -> None: + v = vec[vec[Optional[str]]]() + assert len(v) == 0 + + def test_construct_from_initializer_nested(self) -> None: + v = vec[vec[str]]([vec[str](['xyz'])]) + assert len(v) == 1 + assert v[0][0] == 'xyz' + with self.assertRaises(TypeError): + vec[str](1) + with self.assertRaises(TypeError): + vec[str]([1]) + with self.assertRaises(TypeError): + vec[str]([vec[bytes]()]) + + def test_repr(self) -> None: + assert str(vec[vec[str]]()) == "vec[vec[str]]([])" + assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" + + def test_append_nested(self) -> None: + v = vec[vec[str]]() + vv = append(vec[str](), '1') + v = append(v, vv) + assert str(v) == "vec[vec[str]]([['1']])" + for n in range(2, 10): + vv = append(vec[str](), str(n)) + v = append(v, vv) + assert str(v) == ( + "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" + ) + + def test_append_i64(self) -> None: + v = vec[vec[i64]]() + vv = append(vec[i64](), 1) + v = append(v, vv) + assert str(v) == "vec[vec[i64]]([[1]])" + for n in range(2, 10): + vv = append(vec[i64](), n) + v = append(v, vv) + assert str(v) == ( + "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" + ) + + def test_append_nested_optional(self) -> None: + v = vec[vec[Optional[str]]]() + v = append(v, vec[Optional[str]]()) + v = append(v, vec[Optional[str]](['x', None])) + assert str(v) == "vec[vec[str | None]]([[], ['x', None]])" + + def test_size(self) -> None: + v = vec[vec[str]]() + assert sys.getsizeof(v) == 40 + v = append(v, vec[str]()) + assert sys.getsizeof(v) == 48 # TODO: Why different size? + + def test_get_item(self) -> None: + v = append(vec[vec[str]](), append(vec[str](), 'x')) + assert repr(v[0]) == "vec[str](['x'])" + v = append(v, append(vec[str](), 'y')) + assert repr(v[0]) == "vec[str](['x'])" + assert repr(v[1]) == "vec[str](['y'])" + vv = append(vec[vec[Optional[str]]](), vec[Optional[str]]()) + assert vv[0] == vec[Optional[str]]() + + def test_get_item_error(self) -> None: + v = append(vec[vec[str]](), vec[str](['1'])) + v = append(v, vec[str](['2'])) + v = append(v, vec[str](['3'])) + with self.assertRaises(IndexError): + v[3] + with self.assertRaises(IndexError): + v[4] + n = 2 ** 100 + with self.assertRaises(IndexError): + v[n] + + def test_get_item_negative(self) -> None: + v = vec[vec[str]]([vec[str](['2']), vec[str](['5']), vec[str](['7'])]) + assert v[-1] == vec[str](['7']) + assert v[-2] == vec[str](['5']) + assert v[-3] == vec[str](['2']) + with self.assertRaises(IndexError): + v[-4] + with self.assertRaises(IndexError): + v[-2**62] + + def test_len(self) -> None: + v = vec[vec[str]]() + assert len(v) == 0 + v = append(v, vec[str](['1'])) + assert len(v) == 1 + v = append(v, vec[str]()) + assert len(v) == 2 + for i in range(100): + v = append(v, vec[str](['i'])) + assert len(v) == i + 3 + + def test_set_item(self) -> None: + v = vec[vec[str]]() + v = append(v, vec[str](['1'])) + v = append(v, vec[str](['2'])) + v[0] = vec[str](['3']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['2']) + v[1] = vec[str](['5']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['5']) + + def test_set_item_error_nested(self) -> None: + v = append(vec[vec[str]](), vec[str]()) + v = append(v, vec[str]()) + v = append(v, vec[str]()) + with self.assertRaises(IndexError): + v[3] = vec[str]() + with self.assertRaises(TypeError): + v[0] = 'x' + with self.assertRaises(TypeError): + v[0] = vec[vec[str]]() + with self.assertRaises(TypeError): + v[0] = vec[Optional[str]]() + assert repr(v[0]) == 'vec[str]([])' + + def test_set_item_negative(self) -> None: + v = vec[vec[str]]() + v = append(v, vec[str](['1'])) + v = append(v, vec[str](['2'])) + v[-2] = vec[str](['3']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['2']) + v[-1] = vec[str](['5']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['5']) + with self.assertRaises(IndexError): + v[-3] = vec[str](['5']) + + def test_equality(self) -> None: + print(vec[vec[str]]() == vec[vec[str]]()) + self.assert_eq(vec[vec[str]](), vec[vec[str]]()) + self.assert_eq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['x'])])) + self.assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]()) + self.assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['y'])])) + + def test_equality_different_types(self) -> None: + self.assert_neq(vec[vec[str]](), vec[str]()) + self.assert_neq(vec[vec[str]](), vec[vec[vec[str]]]()) + self.assert_neq(vec[vec[Optional[str]]](), vec[vec[str]]()) + + def assert_eq(self, x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b + + def assert_neq(self, x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + + def test_slicing(self) -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) + assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) + assert v[-2:-1] == vec[vec[str]]([v[3]]) + assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) + assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) + assert v[:] == v + assert v[:] is not v + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) + assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) + assert v[5:100] == vec[vec[str]]([]) + assert v[50:100] == vec[vec[str]]([]) + assert v[-100:-50] == vec[vec[str]]([]) + + def test_slicing_with_step(self) -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) + assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) + assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) + assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) + assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) + with self.assertRaises(ValueError): + v[1:3:0] + + def test_slicing_with_different_item_types(self) -> None: + v = vec[Optional[str]](['x', None, 'y']) + assert v[1:] == vec[Optional[str]]([None, 'y']) + vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) + assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) + + def test_remove(self) -> None: + a = ['4x', '7x', '9x'] + vv = [vec[str]([s]) for s in a] + for i, s in enumerate(a): + v = vec[vec[str]](vv) + v = remove(v, vv[i]) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + # Make sure we have a different object identity + v = remove(v, vec[str]([a[i]])) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + v = remove(v, vv[0]) + v = remove(v, vv[1]) + v = remove(v, vv[2]) + assert v == vec[vec[str]]() + with self.assertRaises(ValueError): + remove(v, vec[str](['4'])) + v = append(v, vec[str](['5'])) + with self.assertRaises(ValueError): + remove(v, vec[str](['7'])) + v = remove(v, vec[str](['5'])) + assert len(v) == 0 + v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) + v = remove(v, vec[str](['x'])) + assert v == v0[1:] + v = remove(v, vec[str](['x'])) + assert v == v0[2:] + v = remove(v, vec[str](['x'])) + assert v == vec[vec[str]]() + vb: Any = vec[bytes]([b'x']) + with self.assertRaises(TypeError): + remove(v, vb) + + def test_pop_last(self) -> None: + v = vec[vec[str]]([vec[str](['4']), vec[str](['5']), vec[str](['9'])]) + v, item = pop(v) + assert item == vec[str](['9']) + assert v == vec[vec[str]]([vec[str](['4']), vec[str](['5'])]) + v, item = pop(v) + assert item == vec[str](['5']) + assert v == vec[vec[str]]([vec[str](['4'])]) + v, item = pop(v) + assert item == vec[str](['4']) + assert v == vec[vec[str]]() + with self.assertRaises(IndexError): + pop(v) + + def test_pop_index(self) -> None: + v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) + v, item = pop(v, 0) + assert item == vec[str](['4']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) + v, item = pop(v, -1) + assert item == vec[str](['22']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) + v, item = pop(v, 1) + assert item == vec[str](['9']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) + + with self.assertRaises(IndexError): + pop(v, 2) + + with self.assertRaises(IndexError): + pop(v, -3) + + v, item = pop(v, -2) + assert item == vec[str](['7']) + assert v == vec[vec[str]]([vec[str](['15'])]) + v, item = pop(v, 0) + assert item == vec[str](['15']) + assert v == vec[vec[str]]() diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index d9f5141b72858..10ca4b8efd659 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -80,6 +80,7 @@ "run-match.test", "run-vecs-i64-interp.test", "run-vecs-t-interp.test", + "run-vecs-nested-interp.test", ] if sys.version_info >= (3, 12): From ca2081390fc73f156e7c088f98d50d1633f46176 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 11:57:31 +0000 Subject: [PATCH 174/208] Refactor test cases to not depend on unittest (still not passing) --- mypyc/test-data/run-vecs-nested-interp.test | 610 ++++++++++---------- 1 file changed, 310 insertions(+), 300 deletions(-) diff --git a/mypyc/test-data/run-vecs-nested-interp.test b/mypyc/test-data/run-vecs-nested-interp.test index c4e8f4f3e2e00..b78ecff51b415 100644 --- a/mypyc/test-data/run-vecs-nested-interp.test +++ b/mypyc/test-data/run-vecs-nested-interp.test @@ -2,306 +2,316 @@ # Test cases for nested vecs, using generic operations (simulates use from interpreted code). import sys -from typing import Optional - -from vecs import vec, append, remove, pop - -from mypy_extensions import i64 - - -class TestVecT(unittest.TestCase): - def test_vec_indexing(self) -> None: - assert type(vec[vec[str]]) is type(vec[str]) - assert type(vec[vec[Optional[str]]]) is type(vec[str]) - assert type(vec[vec[i64]]) is type(vec[str]) - assert type(vec[vec[vec[vec[vec[vec[str]]]]]]) is type(vec[str]) - with self.assertRaises(TypeError): - vec[Optional[vec[str]]] - with self.assertRaises(TypeError): - vec[Optional[vec[Optional[str]]]] - - def test_type_str(self) -> None: - assert str(vec[Optional[str]]) == "" - assert str(vec[vec[str]]) == "" - assert str(vec[vec[i64]]) == "" - assert str(vec[vec[vec[vec[vec[str]]]]]) == "" - assert str( - vec[vec[vec[Optional[str]]]]) == "" - - def test_construct_empty_nested(self) -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v2 = vec[vec[vec[vec[vec[str]]]]]() - assert len(v2) == 0 - v3 = vec[vec[i64]]() - assert len(v3) == 0 - - def test_construct_empty_nested_optional(self) -> None: - v = vec[vec[Optional[str]]]() - assert len(v) == 0 - - def test_construct_from_initializer_nested(self) -> None: - v = vec[vec[str]]([vec[str](['xyz'])]) - assert len(v) == 1 - assert v[0][0] == 'xyz' - with self.assertRaises(TypeError): - vec[str](1) - with self.assertRaises(TypeError): - vec[str]([1]) - with self.assertRaises(TypeError): - vec[str]([vec[bytes]()]) - - def test_repr(self) -> None: - assert str(vec[vec[str]]()) == "vec[vec[str]]([])" - assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" - - def test_append_nested(self) -> None: - v = vec[vec[str]]() - vv = append(vec[str](), '1') +from typing import Any +import typing + +import librt.vecs +import mypy_extensions + +from testutil import assertRaises + +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") +i64: Any = mypy_extensions.i64 + +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + + +def test_vec_indexing() -> None: + assert type(vec[vec[str]]) is type(vec[str]) + assert type(vec[vec[Optional[str]]]) is type(vec[str]) + assert type(vec[vec[i64]]) is type(vec[str]) + assert type(vec[vec[vec[vec[vec[vec[str]]]]]]) is type(vec[str]) + with assertRaises(TypeError): + vec[Optional[vec[str]]] + with assertRaises(TypeError): + vec[Optional[vec[Optional[str]]]] + +def test_type_str() -> None: + assert str(vec[Optional[str]]) == "" + assert str(vec[vec[str]]) == "" + assert str(vec[vec[i64]]) == "" + assert str(vec[vec[vec[vec[vec[str]]]]]) == "" + assert str( + vec[vec[vec[Optional[str]]]]) == "" + +def test_construct_empty_nested() -> None: + v = vec[vec[str]]() + assert len(v) == 0 + v2 = vec[vec[vec[vec[vec[str]]]]]() + assert len(v2) == 0 + v3 = vec[vec[i64]]() + assert len(v3) == 0 + +def test_construct_empty_nested_optional() -> None: + v = vec[vec[Optional[str]]]() + assert len(v) == 0 + +def test_construct_from_initializer_nested() -> None: + v = vec[vec[str]]([vec[str](['xyz'])]) + assert len(v) == 1 + assert v[0][0] == 'xyz' + with assertRaises(TypeError): + vec[str](1) + with assertRaises(TypeError): + vec[str]([1]) + with assertRaises(TypeError): + vec[str]([vec[bytes]()]) + +def test_repr() -> None: + assert str(vec[vec[str]]()) == "vec[vec[str]]([])" + assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" + +def test_append_nested() -> None: + v = vec[vec[str]]() + vv = append(vec[str](), '1') + v = append(v, vv) + assert str(v) == "vec[vec[str]]([['1']])" + for n in range(2, 10): + vv = append(vec[str](), str(n)) v = append(v, vv) - assert str(v) == "vec[vec[str]]([['1']])" - for n in range(2, 10): - vv = append(vec[str](), str(n)) - v = append(v, vv) - assert str(v) == ( - "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" - ) - - def test_append_i64(self) -> None: - v = vec[vec[i64]]() - vv = append(vec[i64](), 1) + assert str(v) == ( + "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" + ) + +def test_append_i64() -> None: + v = vec[vec[i64]]() + vv = append(vec[i64](), 1) + v = append(v, vv) + assert str(v) == "vec[vec[i64]]([[1]])" + for n in range(2, 10): + vv = append(vec[i64](), n) v = append(v, vv) - assert str(v) == "vec[vec[i64]]([[1]])" - for n in range(2, 10): - vv = append(vec[i64](), n) - v = append(v, vv) - assert str(v) == ( - "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" - ) - - def test_append_nested_optional(self) -> None: - v = vec[vec[Optional[str]]]() - v = append(v, vec[Optional[str]]()) - v = append(v, vec[Optional[str]](['x', None])) - assert str(v) == "vec[vec[str | None]]([[], ['x', None]])" - - def test_size(self) -> None: - v = vec[vec[str]]() - assert sys.getsizeof(v) == 40 - v = append(v, vec[str]()) - assert sys.getsizeof(v) == 48 # TODO: Why different size? - - def test_get_item(self) -> None: - v = append(vec[vec[str]](), append(vec[str](), 'x')) - assert repr(v[0]) == "vec[str](['x'])" - v = append(v, append(vec[str](), 'y')) - assert repr(v[0]) == "vec[str](['x'])" - assert repr(v[1]) == "vec[str](['y'])" - vv = append(vec[vec[Optional[str]]](), vec[Optional[str]]()) - assert vv[0] == vec[Optional[str]]() - - def test_get_item_error(self) -> None: - v = append(vec[vec[str]](), vec[str](['1'])) - v = append(v, vec[str](['2'])) - v = append(v, vec[str](['3'])) - with self.assertRaises(IndexError): - v[3] - with self.assertRaises(IndexError): - v[4] - n = 2 ** 100 - with self.assertRaises(IndexError): - v[n] - - def test_get_item_negative(self) -> None: - v = vec[vec[str]]([vec[str](['2']), vec[str](['5']), vec[str](['7'])]) - assert v[-1] == vec[str](['7']) - assert v[-2] == vec[str](['5']) - assert v[-3] == vec[str](['2']) - with self.assertRaises(IndexError): - v[-4] - with self.assertRaises(IndexError): - v[-2**62] - - def test_len(self) -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v = append(v, vec[str](['1'])) - assert len(v) == 1 - v = append(v, vec[str]()) - assert len(v) == 2 - for i in range(100): - v = append(v, vec[str](['i'])) - assert len(v) == i + 3 - - def test_set_item(self) -> None: - v = vec[vec[str]]() - v = append(v, vec[str](['1'])) - v = append(v, vec[str](['2'])) - v[0] = vec[str](['3']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['2']) - v[1] = vec[str](['5']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['5']) - - def test_set_item_error_nested(self) -> None: - v = append(vec[vec[str]](), vec[str]()) - v = append(v, vec[str]()) - v = append(v, vec[str]()) - with self.assertRaises(IndexError): - v[3] = vec[str]() - with self.assertRaises(TypeError): - v[0] = 'x' - with self.assertRaises(TypeError): - v[0] = vec[vec[str]]() - with self.assertRaises(TypeError): - v[0] = vec[Optional[str]]() - assert repr(v[0]) == 'vec[str]([])' - - def test_set_item_negative(self) -> None: - v = vec[vec[str]]() - v = append(v, vec[str](['1'])) - v = append(v, vec[str](['2'])) - v[-2] = vec[str](['3']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['2']) - v[-1] = vec[str](['5']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['5']) - with self.assertRaises(IndexError): - v[-3] = vec[str](['5']) - - def test_equality(self) -> None: - print(vec[vec[str]]() == vec[vec[str]]()) - self.assert_eq(vec[vec[str]](), vec[vec[str]]()) - self.assert_eq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['x'])])) - self.assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]()) - self.assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['y'])])) - - def test_equality_different_types(self) -> None: - self.assert_neq(vec[vec[str]](), vec[str]()) - self.assert_neq(vec[vec[str]](), vec[vec[vec[str]]]()) - self.assert_neq(vec[vec[Optional[str]]](), vec[vec[str]]()) - - def assert_eq(self, x: object, y: object) -> None: - assert x == y - assert y == x - b = x != y - assert not b - b = y != x - assert not b - - def assert_neq(self, x: object, y: object) -> None: - assert x != y - assert y != x - b = x == y - assert not b - b = y == x - assert not b - - def test_slicing(self) -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) - assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) - assert v[-2:-1] == vec[vec[str]]([v[3]]) - assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) - assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) - assert v[:] == v - assert v[:] is not v - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) - assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) - assert v[5:100] == vec[vec[str]]([]) - assert v[50:100] == vec[vec[str]]([]) - assert v[-100:-50] == vec[vec[str]]([]) - - def test_slicing_with_step(self) -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) - assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) - assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) - assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) - assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) - with self.assertRaises(ValueError): - v[1:3:0] - - def test_slicing_with_different_item_types(self) -> None: - v = vec[Optional[str]](['x', None, 'y']) - assert v[1:] == vec[Optional[str]]([None, 'y']) - vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) - assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) - - def test_remove(self) -> None: - a = ['4x', '7x', '9x'] - vv = [vec[str]([s]) for s in a] - for i, s in enumerate(a): - v = vec[vec[str]](vv) - v = remove(v, vv[i]) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) - v = vec[vec[str]](vv) - # Make sure we have a different object identity - v = remove(v, vec[str]([a[i]])) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + assert str(v) == ( + "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" + ) + +def test_append_nested_optional() -> None: + v = vec[vec[Optional[str]]]() + v = append(v, vec[Optional[str]]()) + v = append(v, vec[Optional[str]](['x', None])) + assert str(v) == "vec[vec[str | None]]([[], ['x', None]])" + +def test_size() -> None: + v = vec[vec[str]]() + assert getsizeof(v) == 40 + v = append(v, vec[str]()) + assert getsizeof(v) == 48 # TODO: Why different size? + +def test_get_item() -> None: + v = append(vec[vec[str]](), append(vec[str](), 'x')) + assert repr(v[0]) == "vec[str](['x'])" + v = append(v, append(vec[str](), 'y')) + assert repr(v[0]) == "vec[str](['x'])" + assert repr(v[1]) == "vec[str](['y'])" + vv = append(vec[vec[Optional[str]]](), vec[Optional[str]]()) + assert vv[0] == vec[Optional[str]]() + +def test_get_item_error() -> None: + v = append(vec[vec[str]](), vec[str](['1'])) + v = append(v, vec[str](['2'])) + v = append(v, vec[str](['3'])) + with assertRaises(IndexError): + v[3] + with assertRaises(IndexError): + v[4] + n = 2 ** 100 + with assertRaises(IndexError): + v[n] + +def test_get_item_negative() -> None: + v = vec[vec[str]]([vec[str](['2']), vec[str](['5']), vec[str](['7'])]) + assert v[-1] == vec[str](['7']) + assert v[-2] == vec[str](['5']) + assert v[-3] == vec[str](['2']) + with assertRaises(IndexError): + v[-4] + with assertRaises(IndexError): + v[-2**62] + +def test_len() -> None: + v = vec[vec[str]]() + assert len(v) == 0 + v = append(v, vec[str](['1'])) + assert len(v) == 1 + v = append(v, vec[str]()) + assert len(v) == 2 + for i in range(100): + v = append(v, vec[str](['i'])) + assert len(v) == i + 3 + +def test_set_item() -> None: + v = vec[vec[str]]() + v = append(v, vec[str](['1'])) + v = append(v, vec[str](['2'])) + v[0] = vec[str](['3']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['2']) + v[1] = vec[str](['5']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['5']) + +def test_set_item_error_nested() -> None: + v = append(vec[vec[str]](), vec[str]()) + v = append(v, vec[str]()) + v = append(v, vec[str]()) + with assertRaises(IndexError): + v[3] = vec[str]() + with assertRaises(TypeError): + v[0] = 'x' + with assertRaises(TypeError): + v[0] = vec[vec[str]]() + with assertRaises(TypeError): + v[0] = vec[Optional[str]]() + assert repr(v[0]) == 'vec[str]([])' + +def test_set_item_negative() -> None: + v = vec[vec[str]]() + v = append(v, vec[str](['1'])) + v = append(v, vec[str](['2'])) + v[-2] = vec[str](['3']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['2']) + v[-1] = vec[str](['5']) + assert v[0] == vec[str](['3']) + assert v[1] == vec[str](['5']) + with assertRaises(IndexError): + v[-3] = vec[str](['5']) + +def test_equality() -> None: + assert_eq(vec[vec[str]](), vec[vec[str]]()) + assert_eq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['x'])])) + assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]()) + assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['y'])])) + +def test_equality_different_types() -> None: + assert_neq(vec[vec[str]](), vec[str]()) + assert_neq(vec[vec[str]](), vec[vec[vec[str]]]()) + assert_neq(vec[vec[Optional[str]]](), vec[vec[str]]()) + +def assert_eq(x: object, y: object) -> None: + assert x == y + assert y == x + b = x != y + assert not b + b = y != x + assert not b + +def assert_neq(x: object, y: object) -> None: + assert x != y + assert y != x + b = x == y + assert not b + b = y == x + assert not b + +def test_slicing() -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) + assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) + assert v[-2:-1] == vec[vec[str]]([v[3]]) + assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) + assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) + assert v[:] == v + assert v[:] is not v + assert v[0:5] ==v + assert v[0:5] is not v + assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) + assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) + assert v[5:100] == vec[vec[str]]([]) + assert v[50:100] == vec[vec[str]]([]) + assert v[-100:-50] == vec[vec[str]]([]) + +def test_slicing_with_step() -> None: + v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) + assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) + assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) + assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) + assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) + v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) + assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) + with assertRaises(ValueError): + v[1:3:0] + +def test_slicing_with_different_item_types() -> None: + v = vec[Optional[str]](['x', None, 'y']) + assert v[1:] == vec[Optional[str]]([None, 'y']) + vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) + assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) + +def test_remove() -> None: + a = ['4x', '7x', '9x'] + vv = [vec[str]([s]) for s in a] + for i, s in enumerate(a): v = vec[vec[str]](vv) - v = remove(v, vv[0]) - v = remove(v, vv[1]) - v = remove(v, vv[2]) - assert v == vec[vec[str]]() - with self.assertRaises(ValueError): - remove(v, vec[str](['4'])) - v = append(v, vec[str](['5'])) - with self.assertRaises(ValueError): - remove(v, vec[str](['7'])) - v = remove(v, vec[str](['5'])) - assert len(v) == 0 - v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) - v = remove(v, vec[str](['x'])) - assert v == v0[1:] - v = remove(v, vec[str](['x'])) - assert v == v0[2:] - v = remove(v, vec[str](['x'])) - assert v == vec[vec[str]]() - vb: Any = vec[bytes]([b'x']) - with self.assertRaises(TypeError): - remove(v, vb) - - def test_pop_last(self) -> None: - v = vec[vec[str]]([vec[str](['4']), vec[str](['5']), vec[str](['9'])]) - v, item = pop(v) - assert item == vec[str](['9']) - assert v == vec[vec[str]]([vec[str](['4']), vec[str](['5'])]) - v, item = pop(v) - assert item == vec[str](['5']) - assert v == vec[vec[str]]([vec[str](['4'])]) - v, item = pop(v) - assert item == vec[str](['4']) - assert v == vec[vec[str]]() - with self.assertRaises(IndexError): - pop(v) - - def test_pop_index(self) -> None: - v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) - v, item = pop(v, 0) - assert item == vec[str](['4']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) - v, item = pop(v, -1) - assert item == vec[str](['22']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) - v, item = pop(v, 1) - assert item == vec[str](['9']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) - - with self.assertRaises(IndexError): - pop(v, 2) - - with self.assertRaises(IndexError): - pop(v, -3) - - v, item = pop(v, -2) - assert item == vec[str](['7']) - assert v == vec[vec[str]]([vec[str](['15'])]) - v, item = pop(v, 0) - assert item == vec[str](['15']) - assert v == vec[vec[str]]() + v = remove(v, vv[i]) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + # Make sure we have a different object identity + v = remove(v, vec[str]([a[i]])) + assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) + v = vec[vec[str]](vv) + v = remove(v, vv[0]) + v = remove(v, vv[1]) + v = remove(v, vv[2]) + assert v == vec[vec[str]]() + with assertRaises(ValueError): + remove(v, vec[str](['4'])) + v = append(v, vec[str](['5'])) + with assertRaises(ValueError): + remove(v, vec[str](['7'])) + v = remove(v, vec[str](['5'])) + assert len(v) == 0 + v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) + v = remove(v, vec[str](['x'])) + assert v == v0[1:] + v = remove(v, vec[str](['x'])) + assert v == v0[2:] + v = remove(v, vec[str](['x'])) + assert v == vec[vec[str]]() + vb: Any = vec[bytes]([b'x']) + with assertRaises(TypeError): + remove(v, vb) + +def test_pop_last() -> None: + v = vec[vec[str]]([vec[str](['4']), vec[str](['5']), vec[str](['9'])]) + v, item = pop(v) + assert item == vec[str](['9']) + assert v == vec[vec[str]]([vec[str](['4']), vec[str](['5'])]) + v, item = pop(v) + assert item == vec[str](['5']) + assert v == vec[vec[str]]([vec[str](['4'])]) + v, item = pop(v) + assert item == vec[str](['4']) + assert v == vec[vec[str]]() + with assertRaises(IndexError): + pop(v) + +def test_pop_index() -> None: + v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) + v, item = pop(v, 0) + assert item == vec[str](['4']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) + v, item = pop(v, -1) + assert item == vec[str](['22']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) + v, item = pop(v, 1) + assert item == vec[str](['9']) + assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) + + with assertRaises(IndexError): + pop(v, 2) + + with assertRaises(IndexError): + pop(v, -3) + + v, item = pop(v, -2) + assert item == vec[str](['7']) + assert v == vec[vec[str]]([vec[str](['15'])]) + v, item = pop(v, 0) + assert item == vec[str](['15']) + assert v == vec[vec[str]]() From a4b39156b4a3b7fd59d37d4a0e2bd33f26aa350f Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 12:10:41 +0000 Subject: [PATCH 175/208] Fix pop ref counting --- mypyc/lib-rt/vecs/librt_vecs.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 7d53f91b509f9..69e9ab989a5e9 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -679,6 +679,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if (VecI64_Check(vec)) { VecI64 v = ((VecI64Object *)vec)->vec; + VEC_INCREF(v); VecI64PopResult r; r = VecI64_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -688,6 +689,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecI64_BoxItem(r.f1); } else if (VecU8_Check(vec)) { VecU8 v = ((VecU8Object *)vec)->vec; + VEC_INCREF(v); VecU8PopResult r; r = VecU8_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -697,6 +699,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecU8_BoxItem(r.f1); } else if (VecFloat_Check(vec)) { VecFloat v = ((VecFloatObject *)vec)->vec; + VEC_INCREF(v); VecFloatPopResult r; r = VecFloat_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -706,6 +709,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecFloat_BoxItem(r.f1); } else if (VecI32_Check(vec)) { VecI32 v = ((VecI32Object *)vec)->vec; + VEC_INCREF(v); VecI32PopResult r; r = VecI32_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -715,6 +719,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecI32_BoxItem(r.f1); } else if (VecI16_Check(vec)) { VecI16 v = ((VecI16Object *)vec)->vec; + VEC_INCREF(v); VecI16PopResult r; r = VecI16_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -724,6 +729,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecI16_BoxItem(r.f1); } else if (VecBool_Check(vec)) { VecBool v = ((VecBoolObject *)vec)->vec; + VEC_INCREF(v); VecBoolPopResult r; r = VecBool_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -733,6 +739,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = VecBool_BoxItem(r.f1); } else if (VecT_Check(vec)) { VecT v = ((VecTObject *)vec)->vec; + VEC_INCREF(v); VecTPopResult r; r = VecT_Pop(v, index); if (VEC_IS_ERROR(r.f0)) @@ -745,6 +752,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) result_item1 = r.f1; } else if (VecVec_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; + VEC_INCREF(v); VecNestedPopResult r; r = VecVec_Pop(v, index); if (VEC_IS_ERROR(r.f0)) From 5bef500133544252c3a7ebfba98215c92ec8f04e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 12:27:15 +0000 Subject: [PATCH 176/208] Fixes to reference counting --- mypyc/lib-rt/vecs/vec_nested.c | 4 ++-- mypyc/lib-rt/vecs/vec_t.c | 4 ++-- mypyc/lib-rt/vecs/vec_template.c | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 812deaa63515f..86ac047fb9fac 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -297,8 +297,8 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { Py_XINCREF(items[self.len - 1].buf); } self.len--; - Py_INCREF(self.buf); Py_DECREF(boxed_arg); + // Return the stolen reference without INCREF return self; } } @@ -335,7 +335,7 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { if (v.len > 0) Py_XINCREF(items[v.len - 1].buf); v.len--; - VEC_INCREF(v); + // Return the stolen reference without INCREF result.f0 = v; return result; } diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index f31a9dfacfd3c..db1ac2bb628c4 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -295,7 +295,7 @@ VecT VecT_Remove(VecT v, PyObject *arg) { Py_XINCREF(items[v.len - 1]); } v.len--; - VEC_INCREF(v); + // Return the stolen reference without INCREF return v; } } @@ -331,7 +331,7 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { // to the buffer with a longer length, and they expect a valid reference. Py_XINCREF(items[v.len - 1]); v.len--; - VEC_INCREF(v); + // Return the stolen reference without INCREF result.f0 = v; return result; } diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 8a33e67c50978..28b317e1900bc 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -288,7 +288,7 @@ VEC FUNC(Remove)(VEC v, ITEM_C_TYPE x) { v.buf->items[i] = v.buf->items[i + 1]; } v.len--; - VEC_INCREF(v); + // Return the stolen reference without INCREF return v; } } @@ -322,7 +322,7 @@ NAME(PopResult) FUNC(Pop)(VEC v, Py_ssize_t index) { } v.len--; - VEC_INCREF(v); + // Return the stolen reference without INCREF result.f0 = v; return result; } From 30d9c14e5b06a8d50971a207afb2801f51a90d22 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 13:14:32 +0000 Subject: [PATCH 177/208] Reject Optional[i64] at runtime --- mypyc/lib-rt/vecs/librt_vecs.c | 18 ++++++++++++------ mypyc/test-data/run-vecs-i64-interp.test | 7 +++++++ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 69e9ab989a5e9..86ab3b2ee3410 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -197,6 +197,7 @@ static PyObject *extract_optional_item(PyObject *item) { return NULL; } +// Evaluation vec[] in interpreted code and produce a proxy type object. static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { if (item == (PyObject *)I64TypeObj) { @@ -226,6 +227,17 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (it != NULL) { optional = 1; item = it; + // Check if this is a specialized value type that doesn't support None + if (item == (PyObject *)I64TypeObj || + item == (PyObject *)U8TypeObj || + item == (PyObject *)&PyFloat_Type || + item == (PyObject *)I32TypeObj || + item == (PyObject *)I16TypeObj || + item == (PyObject *)&PyBool_Type) { + PyErr_SetString(PyExc_TypeError, + "vec does not support optional value types (use vec[object] instead)"); + return NULL; + } } if (item->ob_type == &VecProxyType) { if (optional) { @@ -245,27 +257,21 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) if (item == (PyObject *)&VecI64Type) { item_type = VEC_ITEM_TYPE_I64; depth = 1; - // TODO: Check optionals? } else if (item == (PyObject *)&VecU8Type) { item_type = VEC_ITEM_TYPE_U8; depth = 1; - // TODO: Check optionals? } else if (item == (PyObject *)&VecFloatType) { item_type = VEC_ITEM_TYPE_FLOAT; depth = 1; - // TODO: Check optionals? } else if (item == (PyObject *)&VecI32Type) { item_type = VEC_ITEM_TYPE_I32; depth = 1; - // TODO: Check optionals? } else if (item == (PyObject *)&VecI16Type) { item_type = VEC_ITEM_TYPE_I16; depth = 1; - // TODO: Check optionals? } else if (item == (PyObject *)&VecBoolType) { item_type = VEC_ITEM_TYPE_BOOL; depth = 1; - // TODO: Check optionals? } else { item_type = (size_t)item; } diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index dbb60c9e7759e..fbd758afda8f3 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -6,6 +6,7 @@ import sys from typing import Any +import typing import librt.vecs import mypy_extensions @@ -20,11 +21,17 @@ pop: Any = librt.vecs.pop getsizeof: Any = getattr(sys, "getsizeof") i64: Any = mypy_extensions.i64 +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + def test_vec_indexing() -> None: assert type(vec[i64]) is type assert type(vec[i64]) is type(vec[i64]) assert vec[i64] is not vec + with assertRaises(TypeError): + vec[Optional[i64]] # Not supported + def test_type_str() -> None: assert str(vec[i64]) == "" assert repr(vec[i64]) == "" From a7bee4ded88cbd909c68b44474b34b2c011c36aa Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 13:18:53 +0000 Subject: [PATCH 178/208] Fix some TODOs --- mypyc/lib-rt/vecs/librt_vecs.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 86ab3b2ee3410..a7202703245fb 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -833,25 +833,37 @@ PyInit_vecs(void) } I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); - // TODO: Check that it's a type object! if (I64TypeObj == NULL) { return NULL; } + if (!PyType_Check(I64TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i64 is not a type"); + return NULL; + } I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); - // TODO: Check that it's a type object! if (I32TypeObj == NULL) { return NULL; } + if (!PyType_Check(I32TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i32 is not a type"); + return NULL; + } I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); - // TODO: Check that it's a type object! if (I16TypeObj == NULL) { return NULL; } + if (!PyType_Check(I16TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.i16 is not a type"); + return NULL; + } U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); - // TODO: Check that it's a type object! if (U8TypeObj == NULL) { return NULL; } + if (!PyType_Check(U8TypeObj)) { + PyErr_SetString(PyExc_TypeError, "mypy_extensions.u8 is not a type"); + return NULL; + } if (PyType_Ready(&VecGenericType) < 0) return NULL; From dfdf699d062a8ff97c33d97f25f07eb3a8f056f7 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 13:52:59 +0000 Subject: [PATCH 179/208] Improve exception error message --- mypyc/lib-rt/vecs/vec_template.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 28b317e1900bc..393131c229046 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -58,8 +58,7 @@ VEC FUNC(Unbox)(PyObject *obj) { VEC_INCREF(result); // TODO: Should we borrow instead? return result; } else { - // TODO: Better error message - PyErr_SetString(PyExc_TypeError, "vec[i64] expected"); + PyErr_SetString(PyExc_TypeError, "vec[" ITEM_TYPE_STR "] expected"); return vec_error(); } } @@ -222,7 +221,6 @@ static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } static Py_ssize_t vec_length(PyObject *o) { - // TODO: Type check o return ((VEC_OBJECT *)o)->vec.len; } From 2be33ad69b674de9cfaf74384beaf7e5ff732591 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 13:53:17 +0000 Subject: [PATCH 180/208] More tests --- mypyc/test-data/run-vecs-i64-interp.test | 19 +---- mypyc/test-data/run-vecs-misc-interp.test | 87 +++++++++++++++++++++++ mypyc/test/test_run.py | 1 + 3 files changed, 90 insertions(+), 17 deletions(-) create mode 100644 mypyc/test-data/run-vecs-misc-interp.test diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index fbd758afda8f3..1e721f94c783f 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -3,6 +3,8 @@ # # These also act as test cases for specialized vec types in general (e.g. vec[float]), # since much of the implementation is shared between all of them. +# +# There is also run-vecs-misc-interp.test with limited test coverage for vec[float] etc. import sys from typing import Any @@ -24,23 +26,6 @@ i64: Any = mypy_extensions.i64 # Work around test stub limitations Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") -def test_vec_indexing() -> None: - assert type(vec[i64]) is type - assert type(vec[i64]) is type(vec[i64]) - assert vec[i64] is not vec - - with assertRaises(TypeError): - vec[Optional[i64]] # Not supported - -def test_type_str() -> None: - assert str(vec[i64]) == "" - assert repr(vec[i64]) == "" - -def test_construct_empty() -> None: - v = vec[i64]() - assert type(v) is vec[i64] - assert len(v) == 0 - def test_construct_from_initializer() -> None: v = vec[i64]([123]) assert len(v) == 1 diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test new file mode 100644 index 0000000000000..361f3876bc6c4 --- /dev/null +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -0,0 +1,87 @@ +[case testLibrtVecsMiscInterpreted_librt_experimental] +# Test cases for vec[]. using generic operations. +# This simulates use from interpreted code. +# +# All specialized vec types share most of the implementation, so we don't try to +# duplicate all tests in run-vecs-i64-interp.test, which acts also as a more detailed +# test suite for specialized vec types. + +import sys +from typing import Any +import typing + +import librt.vecs +import mypy_extensions + +from testutil import assertRaises + +# Access via Any references to force generic operations +vec: Any = librt.vecs.vec +append: Any = librt.vecs.append +remove: Any = librt.vecs.remove +pop: Any = librt.vecs.pop +getsizeof: Any = getattr(sys, "getsizeof") +u8: Any = mypy_extensions.u8 +i16: Any = mypy_extensions.i16 +i32: Any = mypy_extensions.i32 +i64: Any = mypy_extensions.i64 + +# Work around test stub limitations +Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") + +# TODO: Add some nested vec tests + +ITEM_TYPES = [i64, u8, i16, i32, float, bool] + +def test_vec_indexing() -> None: + for t in ITEM_TYPES: + assert type(vec[t]) is type + t1 = vec[t] + t2 = vec[t] + assert type(vec[t1]) is type(vec[t2]) + _ = [t1, t2] # Ensure t1 and t2 are live here + + assert vec[t] is not vec + if t is not i64: + assert vec[t] is not vec[i64] + else: + assert vec[t] is not vec[i32] + + with assertRaises(TypeError): + vec[Optional[t]] # Not supported + +def test_type_str() -> None: + for t in ITEM_TYPES: + name = t.__name__ + assert str(vec[t]) == f"" + assert repr(vec[t]) == f"" + +def test_construct_empty() -> None: + for t in ITEM_TYPES: + v = vec[t]() + assert type(v) is vec[t] + assert len(v) == 0 + +def test_construct_from_int() -> None: + # All of the non-bool item types support int values 0 and 1 + for t in ITEM_TYPES: + if t is bool: + continue + v = vec[t]([0, 1, 0]) + assert len(v) == 3 + assert v[0] == 0 + assert v[1] == 1 + assert v[2] == 0 + +def test_equality() -> None: + for t1 in ITEM_TYPES: + for t2 in ITEM_TYPES: + if t1 is t2: + assert vec[t1]() == vec[t2]() + else: + assert vec[t1]() != vec[t2]() + +def test_repr() -> None: + for t in ITEM_TYPES: + name = t.__name__ + assert repr(vec[t]()) == f"vec[{name}]([])" diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index 10ca4b8efd659..c6aa034b3f7e8 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -79,6 +79,7 @@ "run-base64.test", "run-match.test", "run-vecs-i64-interp.test", + "run-vecs-misc-interp.test", "run-vecs-t-interp.test", "run-vecs-nested-interp.test", ] From abcfdcf5e4a823472adc0651d040224ddd4e5694 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 13:58:17 +0000 Subject: [PATCH 181/208] Add tests --- mypyc/test-data/run-vecs-misc-interp.test | 263 ++++++++++++++++++++++ 1 file changed, 263 insertions(+) diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test index 361f3876bc6c4..5fba6018a62aa 100644 --- a/mypyc/test-data/run-vecs-misc-interp.test +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -33,6 +33,8 @@ Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") ITEM_TYPES = [i64, u8, i16, i32, float, bool] +# Common tests for all item types + def test_vec_indexing() -> None: for t in ITEM_TYPES: assert type(vec[t]) is type @@ -85,3 +87,264 @@ def test_repr() -> None: for t in ITEM_TYPES: name = t.__name__ assert repr(vec[t]()) == f"vec[{name}]([])" + +# vec[float] tests + +def test_float_construct_from_initializer() -> None: + v = vec[float]([123.5]) + assert len(v) == 1 + assert v[0] == 123.5 + v = vec[float](x + 1 for x in [5, 7.5, -8.5, 10]) + assert len(v) == 4 + assert v[0] == 6 + assert v[1] == 8.5 + assert v[2] == -7.5 + assert v[3] == 11 + with assertRaises(TypeError): + vec[float](1) + with assertRaises(TypeError): + vec[float](['1']) + with assertRaises(TypeError): + vec[float]([None]) + +def test_float_append() -> None: + v = vec[float]() + v = append(v, 1.5) + assert str(v) == "vec[float]([1.5])" + v = append(v, -2) + assert str(v) == "vec[float]([1.5, -2.0])" + +def test_float_get_item() -> None: + v = append(vec[float](), 44.5) + assert v[0] == 44.5 + v = append(v, -56.5) + assert v[0] == 44.5 + assert v[1] == -56.5 + +def test_float_remove() -> None: + a = [4.5, 7, 9] + for x in a: + v = vec[float](a) + v = remove(v, x) + assert v == vec[float]([y for y in a if y != x]) + +def test_float_pop() -> None: + v = vec[float]([4.5, 7, 9.5]) + v, n = pop(v) + assert n == 9.5 + assert v == vec[float]([4.5, 7]) + +# vec[u8] tests + +def test_u8_construct_from_initializer() -> None: + v = vec[u8]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[u8](x for x in [0, 5, 255, 10]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 255 + assert v[3] == 10 + with assertRaises(TypeError): + vec[u8](1) + with assertRaises(TypeError): + vec[u8](['1']) + with assertRaises(TypeError): + vec[u8]([None]) + +def test_u8_append() -> None: + v = vec[u8]() + v = append(v, 5) + assert str(v) == "vec[u8]([5])" + v = append(v, 255) + assert str(v) == "vec[u8]([5, 255])" + +def test_u8_item_overflow() -> None: + with assertRaises(OverflowError): + vec[u8]([256]) + with assertRaises(OverflowError): + vec[u8]([-1]) + with assertRaises(OverflowError): + vec[u8]([2**100]) + with assertRaises(OverflowError): + vec[u8]([-2**100]) + +def test_u8_get_item() -> None: + v = append(vec[u8](), 44) + assert v[0] == 44 + v = append(v, 238) + assert v[0] == 44 + assert v[1] == 238 + +def test_u8_remove() -> None: + a = [0, 7, 255] + for x in a: + v = vec[u8](a) + v = remove(v, x) + assert v == vec[u8]([y for y in a if y != x]) + +def test_u8_pop() -> None: + v = vec[u8]([0, 7, 255]) + v, n = pop(v) + assert n == 255 + assert v == vec[u8]([0, 7]) + +# vec[i16] tests + +def test_i16_construct_from_initializer() -> None: + v = vec[i16]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i16](x for x in [0, 5, 32767, -32768]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 32767 + assert v[3] == -32768 + with assertRaises(TypeError): + vec[i16](1) + with assertRaises(TypeError): + vec[i16](['1']) + with assertRaises(TypeError): + vec[i16]([None]) + +def test_i16_append() -> None: + v = vec[i16]() + v = append(v, 5) + assert str(v) == "vec[i16]([5])" + v = append(v, 32767) + assert str(v) == "vec[i16]([5, 32767])" + +def test_i16_item_overflow() -> None: + with assertRaises(OverflowError): + vec[i16]([32768]) + with assertRaises(OverflowError): + vec[i16]([-32769]) + with assertRaises(OverflowError): + vec[i16]([2**100]) + with assertRaises(OverflowError): + vec[i16]([-2**100]) + +def test_i16_get_item() -> None: + v = append(vec[i16](), 44) + assert v[0] == 44 + v = append(v, -238) + assert v[0] == 44 + assert v[1] == -238 + +def test_i16_remove() -> None: + a = [0, 7, -532, 32767] + for x in a: + v = vec[i16](a) + v = remove(v, x) + assert v == vec[i16]([y for y in a if y != x]) + +def test_i16_pop() -> None: + v = vec[i16]([0, -7, 32767]) + v, n = pop(v) + assert n == 32767 + assert v == vec[i16]([0, -7]) + +# vec[i32] tests + +def test_i32_construct_from_initializer() -> None: + v = vec[i32]([123]) + assert len(v) == 1 + assert v[0] == 123 + v = vec[i32](x for x in [0, 5, 2**31 - 1, -2**31]) + assert len(v) == 4 + assert v[0] == 0 + assert v[1] == 5 + assert v[2] == 2**31 - 1 + assert v[3] == -2**31 + with assertRaises(TypeError): + vec[i32](1) + with assertRaises(TypeError): + vec[i32](['1']) + with assertRaises(TypeError): + vec[i32]([None]) + +def test_i32_append() -> None: + v = vec[i32]() + v = append(v, 5) + assert str(v) == "vec[i32]([5])" + v = append(v, 2**31 - 1) + assert str(v) == "vec[i32]([5, 2147483647])" + +def test_i32_item_overflow() -> None: + with assertRaises(OverflowError): + vec[i32]([2**31]) + with assertRaises(OverflowError): + vec[i32]([-2**31 - 1]) + with assertRaises(OverflowError): + vec[i32]([2**100]) + with assertRaises(OverflowError): + vec[i32]([-2**100]) + +def test_i32_get_item() -> None: + v = append(vec[i32](), 44) + assert v[0] == 44 + v = append(v, -238) + assert v[0] == 44 + assert v[1] == -238 + +def test_i32_remove() -> None: + a = [0, 7, -532, 2**31 - 1] + for x in a: + v = vec[i32](a) + v = remove(v, x) + assert v == vec[i32]([y for y in a if y != x]) + +def test_i32_pop() -> None: + v = vec[i32]([0, -7, 2**31 - 1]) + v, n = pop(v) + assert n == 2**31 - 1 + assert v == vec[i32]([0, -7]) + +# vec[bool] tests + +def test_bool_construct_from_initializer() -> None: + v = vec[bool]([True]) + assert len(v) == 1 + assert v[0] == True + v = vec[bool](x for x in [True, False, True]) + assert len(v) == 3 + assert v[0] is True + assert v[1] is False + assert v[2] is True + with assertRaises(TypeError): + vec[bool](1) + with assertRaises(TypeError): + vec[bool](['1']) + with assertRaises(TypeError): + vec[bool]([1]) + with assertRaises(TypeError): + vec[bool]([None]) + +def test_bool_append() -> None: + v = vec[bool]() + v = append(v, False) + assert str(v) == "vec[bool]([False])" + v = append(v, True) + assert str(v) == "vec[bool]([False, True])" + +def test_bool_get_item() -> None: + v = append(vec[bool](), True) + assert v[0] is True + v = append(v, False) + assert v[0] is True + assert v[1] is False + +def test_bool_remove() -> None: + a = [True, False] + for x in a: + v = vec[bool](a) + v = remove(v, x) + assert v == vec[bool]([y for y in a if y != x]) + +def test_bool_pop() -> None: + v = vec[bool]([True, False, True]) + v, n = pop(v) + assert n is True + assert v == vec[bool]([True, False]) From ee458dc613665ae53e4bfc454f0f32db6c40529a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 14:03:31 +0000 Subject: [PATCH 182/208] Test more --- mypyc/test-data/run-vecs-misc-interp.test | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test index 5fba6018a62aa..9803b325886d3 100644 --- a/mypyc/test-data/run-vecs-misc-interp.test +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -64,20 +64,31 @@ def test_construct_empty() -> None: assert type(v) is vec[t] assert len(v) == 0 -def test_construct_from_int() -> None: - # All of the non-bool item types support int values 0 and 1 +def test_basic_int_operations() -> None: + # All of the non-bool item types support int values 0 to 255 for t in ITEM_TYPES: if t is bool: continue - v = vec[t]([0, 1, 0]) + v = vec[t]([0, 4, 255]) assert len(v) == 3 assert v[0] == 0 - assert v[1] == 1 - assert v[2] == 0 + assert v[1] == 4 + assert v[2] == 255 + assert v[-1] == 255 + assert v[-2] == 4 + assert v[-3] == 0 + for bad_index in 3, -4, 1 << 100, -1 << 100: + with assertRaises(IndexError): + v[bad_index] + with assertRaises(TypeError): + v["0"] + with assertRaises(TypeError): + append(v, "0") def test_equality() -> None: for t1 in ITEM_TYPES: for t2 in ITEM_TYPES: + assert vec[t1]() != [] if t1 is t2: assert vec[t1]() == vec[t2]() else: From edf084e23203bcf66ce42e3931d342f14f9ed376 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 14:10:47 +0000 Subject: [PATCH 183/208] Add test --- mypyc/test-data/run-vecs-misc-interp.test | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test index 9803b325886d3..98989386fe391 100644 --- a/mypyc/test-data/run-vecs-misc-interp.test +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -35,6 +35,12 @@ ITEM_TYPES = [i64, u8, i16, i32, float, bool] # Common tests for all item types +def test_bare_vec_cannot_be_constructed() -> None: + with assertRaises(TypeError): + vec() + with assertRaises(TypeError): + vec([]) + def test_vec_indexing() -> None: for t in ITEM_TYPES: assert type(vec[t]) is type From fa1bb340f1bbfb96570200456bf0b59bcff23956 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 14:31:54 +0000 Subject: [PATCH 184/208] Minor updates --- mypyc/lib-rt/vecs/vec_nested.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 86ac047fb9fac..1907b3f894076 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -405,7 +405,6 @@ VecNestedBuf_dealloc(VecNestedBufObject *self) } static Py_ssize_t vec_ext_length(PyObject *o) { - // TODO: Type check o return ((VecNestedObject *)o)->vec.len; } From 5f14fd4cb4cd1b19dfbf283c04414d56f208f255 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Fri, 23 Jan 2026 17:22:20 +0000 Subject: [PATCH 185/208] Put behind MYPYC_EXPERIMENTAL #define --- mypyc/lib-rt/vecs/librt_vecs.c | 123 ++++++++++++---------- mypyc/lib-rt/vecs/librt_vecs.h | 15 ++- mypyc/lib-rt/vecs/vec_bool.c | 3 + mypyc/lib-rt/vecs/vec_float.c | 3 + mypyc/lib-rt/vecs/vec_i16.c | 3 + mypyc/lib-rt/vecs/vec_i32.c | 3 + mypyc/lib-rt/vecs/vec_i64.c | 3 + mypyc/lib-rt/vecs/vec_nested.c | 3 + mypyc/lib-rt/vecs/vec_t.c | 3 + mypyc/lib-rt/vecs/vec_template.c | 3 + mypyc/lib-rt/vecs/vec_u8.c | 3 + mypyc/test-data/run-vecs-misc-interp.test | 11 ++ 12 files changed, 121 insertions(+), 55 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index a7202703245fb..2aad52218fd10 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -2,6 +2,7 @@ #include #include "librt_vecs.h" +#ifdef MYPYC_EXPERIMENTAL PyTypeObject *I64TypeObj; PyTypeObject *I32TypeObj; @@ -798,21 +799,6 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) return res; } -static PyMethodDef VecsMethods[] = { - {"append", vec_append, METH_VARARGS, "Append a value to the end of a vec"}, - {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value"}, - {"pop", vec_pop, METH_VARARGS, "Remove and return item at index (default last)"}, - {NULL, NULL, 0, NULL} /* Sentinel */ -}; - -static PyModuleDef vecsmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "vecs", - .m_doc = "vecs doc", - .m_size = -1, - .m_methods = VecsMethods, -}; - static VecCapsule Capsule = { &TFeatures, &TExtFeatures, @@ -824,110 +810,139 @@ static VecCapsule Capsule = { &BoolFeatures, }; -PyMODINIT_FUNC -PyInit_vecs(void) +#endif // MYPYC_EXPERIMENTAL + +static PyMethodDef VecsMethods[] = { +#ifdef MYPYC_EXPERIMENTAL + {"append", vec_append, METH_VARARGS, "Append a value to the end of a vec"}, + {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value"}, + {"pop", vec_pop, METH_VARARGS, "Remove and return item at index (default last)"}, +#endif + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static int +librt_vecs_module_exec(PyObject *m) { +#ifdef MYPYC_EXPERIMENTAL PyObject *ext = PyImport_ImportModule("mypy_extensions"); if (ext == NULL) { - return NULL; + return -1; } I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); if (I64TypeObj == NULL) { - return NULL; + return -1; } if (!PyType_Check(I64TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i64 is not a type"); - return NULL; + return -1; } I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); if (I32TypeObj == NULL) { - return NULL; + return -1; } if (!PyType_Check(I32TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i32 is not a type"); - return NULL; + return -1; } I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); if (I16TypeObj == NULL) { - return NULL; + return -1; } if (!PyType_Check(I16TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i16 is not a type"); - return NULL; + return -1; } U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); if (U8TypeObj == NULL) { - return NULL; + return -1; } if (!PyType_Check(U8TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.u8 is not a type"); - return NULL; + return -1; } if (PyType_Ready(&VecGenericType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecProxyType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecTType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecTBufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecNestedType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecNestedBufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI64Type) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI64BufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI32Type) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI32BufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI16Type) < 0) - return NULL; + return -1; if (PyType_Ready(&VecI16BufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecU8Type) < 0) - return NULL; + return -1; if (PyType_Ready(&VecU8BufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecFloatType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecFloatBufType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecBoolType) < 0) - return NULL; + return -1; if (PyType_Ready(&VecBoolBufType) < 0) - return NULL; - - PyObject *m = PyModule_Create(&vecsmodule); - if (m == NULL) - return NULL; + return -1; Py_INCREF(&VecGenericType); if (PyModule_AddObject(m, "vec", (PyObject *)&VecGenericType) < 0) { Py_DECREF(&VecGenericType); - Py_DECREF(m); - return NULL; + return -1; } PyObject *c_api = PyCapsule_New(&Capsule, "librt.vecs._C_API", NULL); if (c_api == NULL) - return NULL; + return -1; if (PyModule_AddObject(m, "_C_API", c_api) < 0) { Py_XDECREF(c_api); Py_DECREF(&VecGenericType); - Py_DECREF(m); - return NULL; + return -1; } Py_DECREF(ext); +#endif + return 0; +} + +static PyModuleDef_Slot librt_vecs_module_slots[] = { + {Py_mod_exec, librt_vecs_module_exec}, +#ifdef Py_MOD_GIL_NOT_USED + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif + {0, NULL} +}; - return m; +static PyModuleDef vecsmodule = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "vecs", + .m_doc = "vecs doc", + .m_size = 0, + .m_methods = VecsMethods, + .m_slots = librt_vecs_module_slots, +}; + +PyMODINIT_FUNC +PyInit_vecs(void) +{ + return PyModuleDef_Init(&vecsmodule); } diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index ce21446fcc2cc..6e649c9c0b3cf 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -1,6 +1,17 @@ #ifndef VEC_H_INCL #define VEC_H_INCL +#ifndef MYPYC_EXPERIMENTAL + +static int +import_librt_vecs(void) +{ + // All librt.vecs features are experimental for now, so don't set up the API here + return 0; +} + +#else // MYPYC_EXPERIMENTAL + #define PY_SSIZE_T_CLEAN #include @@ -810,4 +821,6 @@ int Vec_GenericRemove(Py_ssize_t *len, PyObject **items, PyObject *item); PyObject *Vec_GenericPopWrapper(Py_ssize_t *len, PyObject **items, PyObject *args); PyObject *Vec_GenericPop(Py_ssize_t *len, PyObject **items, Py_ssize_t index); -#endif +#endif // MYPYC_EXPERIMENTAL + +#endif // VEC_H_INCL diff --git a/mypyc/lib-rt/vecs/vec_bool.c b/mypyc/lib-rt/vecs/vec_bool.c index 6217c8f58d48a..197fd8d9d5417 100644 --- a/mypyc/lib-rt/vecs/vec_bool.c +++ b/mypyc/lib-rt/vecs/vec_bool.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecBool #define VEC_TYPE VecBoolType #define VEC_OBJECT VecBoolObject @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecBool_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c index 90cce730029e8..efc02cb189bbc 100644 --- a/mypyc/lib-rt/vecs/vec_float.c +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecFloat #define VEC_TYPE VecFloatType #define VEC_OBJECT VecFloatObject @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecFloat_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i16.c b/mypyc/lib-rt/vecs/vec_i16.c index d4b0dd0e92cd9..658208505f48f 100644 --- a/mypyc/lib-rt/vecs/vec_i16.c +++ b/mypyc/lib-rt/vecs/vec_i16.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecI16 #define VEC_TYPE VecI16Type #define VEC_OBJECT VecI16Object @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecI16_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i32.c b/mypyc/lib-rt/vecs/vec_i32.c index a51b485a61f28..d8efef0b3d579 100644 --- a/mypyc/lib-rt/vecs/vec_i32.c +++ b/mypyc/lib-rt/vecs/vec_i32.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecI32 #define VEC_TYPE VecI32Type #define VEC_OBJECT VecI32Object @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecI32_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index dae31c8048938..d96676688f491 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecI64 #define VEC_TYPE VecI64Type #define VEC_OBJECT VecI64Object @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecI64_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 1907b3f894076..3042aa95a4749 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL // Implementation of nested vec[t], when t is a vec type. // // Examples of types supported: @@ -503,3 +504,5 @@ VecNestedFeatures TExtFeatures = { VecVec_Remove, VecVec_Slice, }; + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index db1ac2bb628c4..133a5218cca4b 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL // Implementation of generic vec[t], when t is a plain type object (possibly optional). // // Examples of types supported: @@ -498,3 +499,5 @@ VecTFeatures TFeatures = { VecT_Remove, VecT_Slice, }; + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 393131c229046..4292e3ffb66d4 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL // NOTE: This file can't be compiled on its own, it must be #included // with certain #defines set, as described below. // @@ -379,3 +380,5 @@ NAME(Features) FEATURES = { FUNC(Remove), FUNC(Slice), }; + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_u8.c b/mypyc/lib-rt/vecs/vec_u8.c index 94976c16cde92..2060f423413c4 100644 --- a/mypyc/lib-rt/vecs/vec_u8.c +++ b/mypyc/lib-rt/vecs/vec_u8.c @@ -1,3 +1,4 @@ +#ifdef MYPYC_EXPERIMENTAL #define VEC VecU8 #define VEC_TYPE VecU8Type #define VEC_OBJECT VecU8Object @@ -15,3 +16,5 @@ #define IS_UNBOX_ERROR VecU8_IsUnboxError #include "vec_template.c" + +#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/test-data/run-vecs-misc-interp.test b/mypyc/test-data/run-vecs-misc-interp.test index 98989386fe391..459822d6776d0 100644 --- a/mypyc/test-data/run-vecs-misc-interp.test +++ b/mypyc/test-data/run-vecs-misc-interp.test @@ -365,3 +365,14 @@ def test_bool_pop() -> None: v, n = pop(v) assert n is True assert v == vec[bool]([True, False]) + +[case testLibrtVecsFeaturesNotAvailableInNonExperimentalBuild_librt] +# This also ensures librt.vecs can be built without experimental features +import librt.vecs + +def test_features_not_available() -> None: + vecs: object = getattr(librt, "vecs") + assert not hasattr(vecs, "vec") + assert not hasattr(vecs, "append") + assert not hasattr(vecs, "remove") + assert not hasattr(vecs, "pop") From 23e31ab882e31d09b57fddf056c6116282d24b5b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:14:58 +0000 Subject: [PATCH 186/208] Refactor: VecProxy -> VecGenericAlias --- mypyc/lib-rt/vecs/librt_vecs.c | 42 +++++++++++++++++----------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 2aad52218fd10..7c9c350f4911e 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -10,7 +10,7 @@ PyTypeObject *I16TypeObj; PyTypeObject *U8TypeObj; -// vec type proxy +// vec generic alias // // Used for the result of generic vec[t] that must preserve knowledge of 't'. // These aren't really types. This only supports constructing instances. @@ -20,16 +20,16 @@ typedef struct { // Can also be one of magic VEC_ITEM_TYPE_* constants size_t item_type; size_t depth; // Number of nested VecNested or VecT types -} VecProxy; +} VecGenericAlias; -static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) +static PyObject *vec_generic_alias_call(PyObject *self, PyObject *args, PyObject *kw) { static char *kwlist[] = {"", NULL}; PyObject *init = NULL; if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { return NULL; } - VecProxy *p = (VecProxy *)self; + VecGenericAlias *p = (VecGenericAlias *)self; if (p->depth == 0) { if (init == NULL) { VecT vec = VecT_New(0, 0, p->item_type); @@ -52,7 +52,7 @@ static PyObject *vec_proxy_call(PyObject *self, PyObject *args, PyObject *kw) } static int -VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) +VecGenericAlias_traverse(VecGenericAlias *self, visitproc visit, void *arg) { if (!Vec_IsMagicItemType(self->item_type)) Py_VISIT((PyObject *)(self->item_type & ~1)); @@ -60,7 +60,7 @@ VecProxy_traverse(VecProxy *self, visitproc visit, void *arg) } static void -VecProxy_dealloc(VecProxy *self) +VecGenericAlias_dealloc(VecGenericAlias *self) { if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { Py_DECREF((PyObject *)(self->item_type & ~1)); @@ -113,7 +113,7 @@ PyObject *Vec_TypeToStr(size_t item_type, size_t depth) { return result; } -PyObject *VecProxy_repr(PyObject *self) { +PyObject *VecGenericAlias_repr(PyObject *self) { PyObject *l = NULL; PyObject *prefix = NULL; PyObject *suffix = NULL; @@ -135,7 +135,7 @@ PyObject *VecProxy_repr(PyObject *self) { if (PyList_Append(l, prefix) < 0) goto error; - VecProxy *v = (VecProxy *)self; + VecGenericAlias *v = (VecGenericAlias *)self; type_str = Vec_TypeToStr(v->item_type, v->depth); if (type_str == NULL) goto error; @@ -153,16 +153,16 @@ PyObject *VecProxy_repr(PyObject *self) { return result; } -PyTypeObject VecProxyType = { +PyTypeObject VecGenericAliasType = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec_proxy", - .tp_basicsize = sizeof(VecProxy), + .tp_name = "vec_generic_alias", + .tp_basicsize = sizeof(VecGenericAlias), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_call = vec_proxy_call, - .tp_traverse = (traverseproc)VecProxy_traverse, - .tp_dealloc = (destructor)VecProxy_dealloc, - .tp_repr = (reprfunc)VecProxy_repr, + .tp_call = vec_generic_alias_call, + .tp_traverse = (traverseproc)VecGenericAlias_traverse, + .tp_dealloc = (destructor)VecGenericAlias_dealloc, + .tp_repr = (reprfunc)VecGenericAlias_repr, }; @@ -198,7 +198,7 @@ static PyObject *extract_optional_item(PyObject *item) { return NULL; } -// Evaluation vec[] in interpreted code and produce a proxy type object. +// Evaluation vec[] in interpreted code and produce a generic alias object. static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { if (item == (PyObject *)I64TypeObj) { @@ -240,12 +240,12 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) return NULL; } } - if (item->ob_type == &VecProxyType) { + if (item->ob_type == &VecGenericAliasType) { if (optional) { PyErr_SetString(PyExc_TypeError, "optional type not expected in vec[...]"); return NULL; } - VecProxy *p = (VecProxy *)item; + VecGenericAlias *p = (VecGenericAlias *)item; item_type = p->item_type; depth = p->depth + 1; } else if (!PyObject_TypeCheck(item, &PyType_Type)) { @@ -282,8 +282,8 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) ((PyTypeObject *)item)->tp_name); return NULL; } - VecProxy *p; - p = PyObject_GC_New(VecProxy, &VecProxyType); + VecGenericAlias *p; + p = PyObject_GC_New(VecGenericAlias, &VecGenericAliasType); if (p == NULL) return NULL; Py_INCREF(item); @@ -865,7 +865,7 @@ librt_vecs_module_exec(PyObject *m) if (PyType_Ready(&VecGenericType) < 0) return -1; - if (PyType_Ready(&VecProxyType) < 0) + if (PyType_Ready(&VecGenericAliasType) < 0) return -1; if (PyType_Ready(&VecTType) < 0) From 491ca642901561132e7a9057d74f04b1a28fa40c Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:18:19 +0000 Subject: [PATCH 187/208] Remove TODO comment --- mypyc/lib-rt/vecs/vec_t.c | 1 - 1 file changed, 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 133a5218cca4b..706b598faacba 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -401,7 +401,6 @@ VecTBuf_dealloc(VecTBufObject *self) } static Py_ssize_t vec_length(PyObject *o) { - // TODO: Type check o return ((VecTObject *)o)->vec.len; } From 3bed433d296a0ba78b2a2b3edc116794edccf736 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:23:45 +0000 Subject: [PATCH 188/208] Refactor: rename VecGeneric -> Vec --- mypyc/lib-rt/vecs/librt_vecs.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 7c9c350f4911e..a00bf4bc74cda 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -172,7 +172,7 @@ PyTypeObject VecGenericAliasType = { typedef struct { PyObject_HEAD -} VecGeneric; +} Vec; static PyObject *extract_optional_item(PyObject *item) { PyObject *args = PyObject_GetAttrString(item, "__args__"); @@ -299,10 +299,10 @@ static PyMethodDef vec_methods[] = { {NULL, NULL, 0, NULL}, /* Sentinel */ }; -PyTypeObject VecGenericType = { +PyTypeObject VecType = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vec", - .tp_basicsize = sizeof(VecGeneric), + .tp_basicsize = sizeof(Vec), .tp_itemsize = 0, .tp_flags = Py_TPFLAGS_DEFAULT, .tp_methods = vec_methods, @@ -863,7 +863,7 @@ librt_vecs_module_exec(PyObject *m) return -1; } - if (PyType_Ready(&VecGenericType) < 0) + if (PyType_Ready(&VecType) < 0) return -1; if (PyType_Ready(&VecGenericAliasType) < 0) return -1; @@ -903,9 +903,9 @@ librt_vecs_module_exec(PyObject *m) if (PyType_Ready(&VecBoolBufType) < 0) return -1; - Py_INCREF(&VecGenericType); - if (PyModule_AddObject(m, "vec", (PyObject *)&VecGenericType) < 0) { - Py_DECREF(&VecGenericType); + Py_INCREF(&VecType); + if (PyModule_AddObject(m, "vec", (PyObject *)&VecType) < 0) { + Py_DECREF(&VecType); return -1; } @@ -915,7 +915,7 @@ librt_vecs_module_exec(PyObject *m) if (PyModule_AddObject(m, "_C_API", c_api) < 0) { Py_XDECREF(c_api); - Py_DECREF(&VecGenericType); + Py_DECREF(&VecType); return -1; } From cc9841ac3d1ca8246531aad729fc2b6fb71c156a Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:44:07 +0000 Subject: [PATCH 189/208] Add comments --- mypyc/lib-rt/vecs/librt_vecs.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index a00bf4bc74cda..9364c43aedbdf 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -12,8 +12,8 @@ PyTypeObject *U8TypeObj; // vec generic alias // -// Used for the result of generic vec[t] that must preserve knowledge of 't'. -// These aren't really types. This only supports constructing instances. +// Used for the result of vec[t] (indexing) in interpeted context that must preserve +// knowledge of 't'. These aren't real types. This only supports constructing instances. typedef struct { PyObject_HEAD // Tagged pointer to PyTypeObject *, lowest bit set for optional item type @@ -308,6 +308,8 @@ PyTypeObject VecType = { .tp_methods = vec_methods, }; +// Common helpers + PyObject *Vec_GenericRepr(PyObject *vec, size_t item_type, size_t depth, int verbose) { PyObject *l = NULL; PyObject *prefix = NULL; From a99a1cbeb87b1550ed2bc50f030bb6dd04f17f81 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:44:23 +0000 Subject: [PATCH 190/208] Update docstrings --- mypyc/lib-rt/vecs/librt_vecs.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 9364c43aedbdf..9df6ddf920dba 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -817,8 +817,8 @@ static VecCapsule Capsule = { static PyMethodDef VecsMethods[] = { #ifdef MYPYC_EXPERIMENTAL {"append", vec_append, METH_VARARGS, "Append a value to the end of a vec"}, - {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value"}, - {"pop", vec_pop, METH_VARARGS, "Remove and return item at index (default last)"}, + {"remove", vec_remove, METH_VARARGS, "Remove first occurrence of value from a vec"}, + {"pop", vec_pop, METH_VARARGS, "Remove and return vec item at index (default last)"}, #endif {NULL, NULL, 0, NULL} /* Sentinel */ }; From cc0f79307df28fca604fa4fc5052bc702dff74b6 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:50:24 +0000 Subject: [PATCH 191/208] Refactor --- mypyc/lib-rt/vecs/librt_vecs.h | 9 +++++---- mypyc/lib-rt/vecs/vec_nested.c | 14 +++++++------- mypyc/lib-rt/vecs/vec_t.c | 14 +++++++------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 6e649c9c0b3cf..ba1bbe7992bad 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -451,9 +451,9 @@ typedef struct { VecBoolFeatures *bool_; } VecCapsule; -#define BUF_SIZE(b) ((b)->ob_base.ob_size) -#define ITEM_TYPE(t) ((PyTypeObject *)((t) & ~1)) -#define BUF_ITEM_TYPE(b) ITEM_TYPE((b)->item_type) +#define VEC_BUF_SIZE(b) ((b)->ob_base.ob_size) +#define VEC_ITEM_TYPE(t) ((PyTypeObject *)((t) & ~1)) +#define VEC_BUF_ITEM_TYPE(b) VEC_ITEM_TYPE((b)->item_type) #define VEC_CAP(v) ((v).buf->ob_base.ob_size) #define VEC_IS_ERROR(v) ((v).len < 0) #define VEC_DECREF(v) Py_XDECREF((v).buf) @@ -479,6 +479,7 @@ extern PyTypeObject VecBoolType; extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; +// Type objects correponding the 'i64', 'i32', 'i16, and 'u8' types extern PyTypeObject *I64TypeObj; extern PyTypeObject *I32TypeObj; extern PyTypeObject *I16TypeObj; @@ -685,7 +686,7 @@ static inline int VecT_Check(PyObject *o) { } static inline int VecT_ItemCheck(VecT v, PyObject *item, size_t item_type) { - if (PyObject_TypeCheck(item, ITEM_TYPE(item_type))) { + if (PyObject_TypeCheck(item, VEC_ITEM_TYPE(item_type))) { return 1; } else if ((item_type & 1) && item == Py_None) { return 1; diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 3042aa95a4749..17fa9ef44e97a 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -28,7 +28,7 @@ static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { buf->item_type = item_type; buf->depth = depth; if (!Vec_IsMagicItemType(item_type)) - Py_INCREF(BUF_ITEM_TYPE(buf)); + Py_INCREF(VEC_BUF_ITEM_TYPE(buf)); VecNested res = { .buf = buf }; PyObject_GC_Track(buf); return res; @@ -369,8 +369,8 @@ static int VecNestedBuf_traverse(VecNestedBufObject *self, visitproc visit, void *arg) { if (!Vec_IsMagicItemType(self->item_type)) - Py_VISIT(BUF_ITEM_TYPE(self)); - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_VISIT(VEC_BUF_ITEM_TYPE(self)); + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_VISIT(self->items[i].buf); } return 0; @@ -380,10 +380,10 @@ static int VecNestedBuf_clear(VecNestedBufObject *self) { if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { - Py_DECREF(BUF_ITEM_TYPE(self)); + Py_DECREF(VEC_BUF_ITEM_TYPE(self)); self->item_type = 0; } - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_CLEAR(self->items[i].buf); } return 0; @@ -395,10 +395,10 @@ VecNestedBuf_dealloc(VecNestedBufObject *self) PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecNestedBuf_dealloc) if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { - Py_DECREF(BUF_ITEM_TYPE(self)); + Py_DECREF(VEC_BUF_ITEM_TYPE(self)); self->item_type = 0; } - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_CLEAR(self->items[i].buf); } Py_TYPE(self)->tp_free((PyObject *)self); diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index 706b598faacba..e21089ff700b3 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -22,7 +22,7 @@ static inline VecTBufObject *alloc_buf(Py_ssize_t size, size_t item_type) { if (buf == NULL) return NULL; buf->item_type = item_type; - Py_INCREF(BUF_ITEM_TYPE(buf)); + Py_INCREF(VEC_BUF_ITEM_TYPE(buf)); PyObject_GC_Track(buf); return buf; } @@ -364,8 +364,8 @@ VecT_dealloc(VecTObject *self) static int VecTBuf_traverse(VecTBufObject *self, visitproc visit, void *arg) { - Py_VISIT(BUF_ITEM_TYPE(self)); - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + Py_VISIT(VEC_BUF_ITEM_TYPE(self)); + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_VISIT(self->items[i]); } return 0; @@ -375,10 +375,10 @@ static int VecTBuf_clear(VecTBufObject *self) { if (self->item_type) { - Py_DECREF(BUF_ITEM_TYPE(self)); + Py_DECREF(VEC_BUF_ITEM_TYPE(self)); self->item_type = 0; } - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_CLEAR(self->items[i]); } return 0; @@ -390,10 +390,10 @@ VecTBuf_dealloc(VecTBufObject *self) PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecTBuf_dealloc) if (self->item_type) { - Py_DECREF(BUF_ITEM_TYPE(self)); + Py_DECREF(VEC_BUF_ITEM_TYPE(self)); self->item_type = 0; } - for (Py_ssize_t i = 0; i < BUF_SIZE(self); i++) { + for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { Py_CLEAR(self->items[i]); } Py_TYPE(self)->tp_free((PyObject *)self); From d007176a4f55887fba1b515b7c7ee3cdb50a7a6b Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:55:10 +0000 Subject: [PATCH 192/208] Refactor: change VecVec prefix in names to VecNested --- mypyc/lib-rt/vecs/librt_vecs.c | 30 ++++++++-------- mypyc/lib-rt/vecs/librt_vecs.h | 20 +++++------ mypyc/lib-rt/vecs/vec_nested.c | 62 +++++++++++++++++----------------- 3 files changed, 56 insertions(+), 56 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 9df6ddf920dba..7824460db5f1a 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -41,12 +41,12 @@ static PyObject *vec_generic_alias_call(PyObject *self, PyObject *args, PyObject } } else { if (init == NULL) { - VecNested vec = VecVec_New(0, 0, p->item_type, p->depth); + VecNested vec = VecNested_New(0, 0, p->item_type, p->depth); if (VEC_IS_ERROR(vec)) return NULL; - return VecVec_Box(vec); + return VecNested_Box(vec); } else { - return VecVec_FromIterable(p->item_type, p->depth, init); + return VecNested_FromIterable(p->item_type, p->depth, init); } } } @@ -557,16 +557,16 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecT_Box(v, v.buf->item_type); - } else if (VecVec_Check(vec)) { + } else if (VecNested_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; VecNestedBufItem vecitem; - if (VecVec_UnboxItem(v, item, &vecitem) < 0) + if (VecNested_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = VecVec_Append(v, vecitem); + v = VecNested_Append(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; - return VecVec_Box(v); + return VecNested_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", Py_TYPE(vec)->tp_name); @@ -658,16 +658,16 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecT_Box(v, v.buf->item_type); - } else if (VecVec_Check(vec)) { + } else if (VecNested_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; VecNestedBufItem vecitem; - if (VecVec_UnboxItem(v, item, &vecitem) < 0) + if (VecNested_UnboxItem(v, item, &vecitem) < 0) return NULL; VEC_INCREF(v); - v = VecVec_Remove(v, vecitem); + v = VecNested_Remove(v, vecitem); if (VEC_IS_ERROR(v)) return NULL; - return VecVec_Box(v); + return VecNested_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", Py_TYPE(vec)->tp_name); @@ -759,20 +759,20 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) return NULL; } result_item1 = r.f1; - } else if (VecVec_Check(vec)) { + } else if (VecNested_Check(vec)) { VecNested v = ((VecNestedObject *)vec)->vec; VEC_INCREF(v); VecNestedPopResult r; - r = VecVec_Pop(v, index); + r = VecNested_Pop(v, index); if (VEC_IS_ERROR(r.f0)) return NULL; - result_item0 = VecVec_Box(r.f0); + result_item0 = VecNested_Box(r.f0); if (result_item0 == NULL) { Py_DECREF(r.f0.buf); Py_DECREF(r.f1.buf); return NULL; } - result_item1 = VecVec_BoxItem(r.f0, r.f1); + result_item1 = VecNested_BoxItem(r.f0, r.f1); if (result_item1 == NULL) { Py_DECREF(result_item0); Py_DECREF(r.f1.buf); diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index ba1bbe7992bad..4bd24a49900f7 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -706,20 +706,20 @@ VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); // Nested vec operations -static inline int VecVec_Check(PyObject *o) { +static inline int VecNested_Check(PyObject *o) { return o->ob_type == &VecNestedType; } -VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); -PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable); -PyObject *VecVec_Box(VecNested); -VecNested VecVec_Append(VecNested vec, VecNestedBufItem x); -VecNested VecVec_Remove(VecNested vec, VecNestedBufItem x); -VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index); +VecNested VecNested_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); +PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *iterable); +PyObject *VecNested_Box(VecNested); +VecNested VecNested_Append(VecNested vec, VecNestedBufItem x); +VecNested VecNested_Remove(VecNested vec, VecNestedBufItem x); +VecNestedPopResult VecNested_Pop(VecNested v, Py_ssize_t index); // Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. // Return a *borrowed* reference. -static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem *unboxed) { +static inline int VecNested_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem *unboxed) { size_t depth = v.buf->depth; if (depth == 1) { if (item->ob_type == &VecTType) { @@ -774,14 +774,14 @@ static inline int VecVec_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem return -1; } -static inline PyObject *VecVec_BoxItem(VecNested v, VecNestedBufItem item) { +static inline PyObject *VecNested_BoxItem(VecNested v, VecNestedBufItem item) { if (item.len < 0) Py_RETURN_NONE; Py_XINCREF(item.buf); if (v.buf->depth > 1) { // Item is a nested vec VecNested v = { .len = item.len, .buf = (VecNestedBufObject *)item.buf }; - return VecVec_Box(v); + return VecNested_Box(v); } else { // Item is a non-nested vec size_t item_type = v.buf->item_type; diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 17fa9ef44e97a..79dd0a7080828 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -16,7 +16,7 @@ static inline VecNested vec_error() { } static inline PyObject *box_vec_item_by_index(VecNested v, Py_ssize_t index) { - return VecVec_BoxItem(v, v.buf->items[index]); + return VecNested_BoxItem(v, v.buf->items[index]); } // Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the @@ -35,7 +35,7 @@ static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { } // Box a nested vec value, stealing 'vec'. On error, decref 'vec'. -PyObject *VecVec_Box(VecNested vec) { +PyObject *VecNested_Box(VecNested vec) { VecNestedObject *obj = PyObject_GC_New(VecNestedObject, &VecNestedType); if (obj == NULL) { VEC_DECREF(vec); @@ -46,7 +46,7 @@ PyObject *VecVec_Box(VecNested vec) { return (PyObject *)obj; } -VecNested VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { +VecNested VecNested_Unbox(PyObject *obj, size_t item_type, size_t depth) { if (obj->ob_type == &VecNestedType) { VecNested result = ((VecNestedObject *)obj)->vec; if (result.buf->item_type == item_type && result.buf->depth == depth) { @@ -59,11 +59,11 @@ VecNested VecVec_Unbox(PyObject *obj, size_t item_type, size_t depth) { return vec_error(); } -VecNested VecVec_ConvertFromNested(VecNestedBufItem item) { +VecNested VecNested_ConvertFromNested(VecNestedBufItem item) { return (VecNested) { item.len, (VecNestedBufObject *)item.buf }; } -VecNested VecVec_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { +VecNested VecNested_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { if (cap < size) cap = size; VecNested vec = vec_alloc(cap, item_type, depth); @@ -94,7 +94,7 @@ static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { } } -VecNested VecVec_Slice(VecNested vec, int64_t start, int64_t end) { +VecNested VecNested_Slice(VecNested vec, int64_t start, int64_t end) { if (start < 0) start += vec.len; if (end < 0) @@ -150,7 +150,7 @@ static PyObject *vec_subscript(PyObject *self, PyObject *item) { res.buf->items[i] = item; j += step; } - return VecVec_Box(res); + return VecNested_Box(res); } else { PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", item->ob_type->tp_name); @@ -165,7 +165,7 @@ static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { } if ((size_t)i < (size_t)v.len) { VecNestedBufItem item; - if (VecVec_UnboxItem(v, o, &item) < 0) + if (VecNested_UnboxItem(v, o, &item) < 0) return -1; VEC_INCREF(item); VEC_DECREF(v.buf->items[i]); @@ -222,7 +222,7 @@ PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { } // Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. -VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { +VecNested VecNested_Append(VecNested vec, VecNestedBufItem x) { Py_ssize_t cap = VEC_CAP(vec); VEC_INCREF(x); if (vec.len < cap) { @@ -254,10 +254,10 @@ VecNested VecVec_Append(VecNested vec, VecNestedBufItem x) { } // Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. -VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { +VecNested VecNested_Remove(VecNested self, VecNestedBufItem arg) { VecNestedBufItem *items = self.buf->items; - PyObject *boxed_arg = VecVec_BoxItem(self, arg); + PyObject *boxed_arg = VecNested_BoxItem(self, arg); if (boxed_arg == NULL) { // The input self is being consumed/stolen by this function, so on error // we must decref it to avoid leaking the buffer. @@ -312,7 +312,7 @@ VecNested VecVec_Remove(VecNested self, VecNestedBufItem arg) { } // Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. -VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { +VecNestedPopResult VecNested_Pop(VecNested v, Py_ssize_t index) { VecNestedPopResult result; if (index < 0) @@ -342,24 +342,24 @@ VecNestedPopResult VecVec_Pop(VecNested v, Py_ssize_t index) { } static int -VecVec_traverse(VecTObject *self, visitproc visit, void *arg) +VecNested_traverse(VecTObject *self, visitproc visit, void *arg) { Py_VISIT(self->vec.buf); return 0; } static int -VecVec_clear(VecTObject *self) +VecNested_clear(VecTObject *self) { Py_CLEAR(self->vec.buf); return 0; } static void -VecVec_dealloc(VecTObject *self) +VecNested_dealloc(VecTObject *self) { PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecVec_dealloc) + Py_TRASHCAN_BEGIN(self, VecNested_dealloc) Py_CLEAR(self->vec.buf); Py_TYPE(self)->tp_free((PyObject *)self); Py_TRASHCAN_END @@ -444,9 +444,9 @@ PyTypeObject VecNestedType = { .tp_basicsize = sizeof(VecNestedObject) - sizeof(PyObject *), .tp_itemsize = sizeof(PyObject *), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecVec_traverse, - .tp_clear = (inquiry)VecVec_clear, - .tp_dealloc = (destructor)VecVec_dealloc, + .tp_traverse = (traverseproc)VecNested_traverse, + .tp_clear = (inquiry)VecNested_clear, + .tp_dealloc = (destructor)VecNested_dealloc, //.tp_free = PyObject_GC_Del, .tp_repr = (reprfunc)vec_repr, .tp_as_sequence = &VecNestedSequence, @@ -456,7 +456,7 @@ PyTypeObject VecNestedType = { // TODO: free }; -PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { +PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { VecNested v = vec_alloc(0, item_type, depth); if (VEC_IS_ERROR(v)) return NULL; @@ -470,13 +470,13 @@ PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable PyObject *item; while ((item = PyIter_Next(iter)) != NULL) { VecNestedBufItem vecitem; - if (VecVec_UnboxItem(v, item, &vecitem) < 0) { + if (VecNested_UnboxItem(v, item, &vecitem) < 0) { Py_DECREF(iter); VEC_DECREF(v); Py_DECREF(item); return NULL; } - v = VecVec_Append(v, vecitem); + v = VecNested_Append(v, vecitem); Py_DECREF(item); if (VEC_IS_ERROR(v)) { Py_DECREF(iter); @@ -489,20 +489,20 @@ PyObject *VecVec_FromIterable(size_t item_type, size_t depth, PyObject *iterable VEC_DECREF(v); return NULL; } - return VecVec_Box(v); + return VecNested_Box(v); } VecNestedFeatures TExtFeatures = { &VecNestedType, &VecNestedBufType, - VecVec_New, - VecVec_Box, - VecVec_Unbox, - VecVec_ConvertFromNested, - VecVec_Append, - VecVec_Pop, - VecVec_Remove, - VecVec_Slice, + VecNested_New, + VecNested_Box, + VecNested_Unbox, + VecNested_ConvertFromNested, + VecNested_Append, + VecNested_Pop, + VecNested_Remove, + VecNested_Slice, }; #endif // MYPYC_EXPERIMENTAL From 17507d1dc22dae2f135a66e167151f7187781bd4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 11:58:11 +0000 Subject: [PATCH 193/208] Refactor --- mypyc/lib-rt/vecs/librt_vecs.c | 2 +- mypyc/lib-rt/vecs/librt_vecs.h | 2 +- mypyc/lib-rt/vecs/vec_nested.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 7824460db5f1a..35efadf1bef80 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -803,7 +803,7 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) static VecCapsule Capsule = { &TFeatures, - &TExtFeatures, + &NestedFeatures, &I64Features, &I32Features, &I16Features, diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 4bd24a49900f7..eaf0271735cbf 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -492,7 +492,7 @@ extern VecU8Features U8Features; extern VecFloatFeatures FloatFeatures; extern VecBoolFeatures BoolFeatures; extern VecTFeatures TFeatures; -extern VecNestedFeatures TExtFeatures; +extern VecNestedFeatures NestedFeatures; static inline int Vec_CheckFloatError(PyObject *o) { if (PyFloat_Check(o)) { diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 79dd0a7080828..591e5c9e85ca2 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -492,7 +492,7 @@ PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *itera return VecNested_Box(v); } -VecNestedFeatures TExtFeatures = { +VecNestedFeatures NestedFeatures = { &VecNestedType, &VecNestedBufType, VecNested_New, From 61d6e146ddc09a96707d0243eb79d0827d7182e5 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 13:09:21 +0000 Subject: [PATCH 194/208] Minor tweaks --- mypyc/lib-rt/vecs/vec_nested.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 591e5c9e85ca2..569e038f010cd 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -342,21 +342,21 @@ VecNestedPopResult VecNested_Pop(VecNested v, Py_ssize_t index) { } static int -VecNested_traverse(VecTObject *self, visitproc visit, void *arg) +VecNested_traverse(VecNestedObject *self, visitproc visit, void *arg) { Py_VISIT(self->vec.buf); return 0; } static int -VecNested_clear(VecTObject *self) +VecNested_clear(VecNestedObject *self) { Py_CLEAR(self->vec.buf); return 0; } static void -VecNested_dealloc(VecTObject *self) +VecNested_dealloc(VecNestedObject *self) { PyObject_GC_UnTrack(self); Py_TRASHCAN_BEGIN(self, VecNested_dealloc) From 1cca0c66aaf3a9ff6778ad106f3dbeb83effaf00 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 13:37:57 +0000 Subject: [PATCH 195/208] Rename more --- mypyc/lib-rt/vecs/librt_vecs.c | 48 +++++++++++++++++----------------- mypyc/lib-rt/vecs/librt_vecs.h | 8 +++--- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 35efadf1bef80..02edee1cee01a 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -4,10 +4,10 @@ #ifdef MYPYC_EXPERIMENTAL -PyTypeObject *I64TypeObj; -PyTypeObject *I32TypeObj; -PyTypeObject *I16TypeObj; -PyTypeObject *U8TypeObj; +PyTypeObject *LibRTVecs_I64TypeObj; +PyTypeObject *LibRTVecs_I32TypeObj; +PyTypeObject *LibRTVecs_I16TypeObj; +PyTypeObject *LibRTVecs_U8TypeObj; // vec generic alias @@ -201,19 +201,19 @@ static PyObject *extract_optional_item(PyObject *item) { // Evaluation vec[] in interpreted code and produce a generic alias object. static PyObject *vec_class_getitem(PyObject *type, PyObject *item) { - if (item == (PyObject *)I64TypeObj) { + if (item == (PyObject *)LibRTVecs_I64TypeObj) { Py_INCREF(&VecI64Type); return (PyObject *)&VecI64Type; - } else if (item == (PyObject *)U8TypeObj) { + } else if (item == (PyObject *)LibRTVecs_U8TypeObj) { Py_INCREF(&VecU8Type); return (PyObject *)&VecU8Type; } else if (item == (PyObject *)&PyFloat_Type) { Py_INCREF(&VecFloatType); return (PyObject *)&VecFloatType; - } else if (item == (PyObject *)I32TypeObj) { + } else if (item == (PyObject *)LibRTVecs_I32TypeObj) { Py_INCREF(&VecI32Type); return (PyObject *)&VecI32Type; - } else if (item == (PyObject *)I16TypeObj) { + } else if (item == (PyObject *)LibRTVecs_I16TypeObj) { Py_INCREF(&VecI16Type); return (PyObject *)&VecI16Type; } else if (item == (PyObject *)&PyBool_Type) { @@ -229,11 +229,11 @@ static PyObject *vec_class_getitem(PyObject *type, PyObject *item) optional = 1; item = it; // Check if this is a specialized value type that doesn't support None - if (item == (PyObject *)I64TypeObj || - item == (PyObject *)U8TypeObj || + if (item == (PyObject *)LibRTVecs_I64TypeObj || + item == (PyObject *)LibRTVecs_U8TypeObj || item == (PyObject *)&PyFloat_Type || - item == (PyObject *)I32TypeObj || - item == (PyObject *)I16TypeObj || + item == (PyObject *)LibRTVecs_I32TypeObj || + item == (PyObject *)LibRTVecs_I16TypeObj || item == (PyObject *)&PyBool_Type) { PyErr_SetString(PyExc_TypeError, "vec does not support optional value types (use vec[object] instead)"); @@ -832,35 +832,35 @@ librt_vecs_module_exec(PyObject *m) return -1; } - I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); - if (I64TypeObj == NULL) { + LibRTVecs_I64TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i64"); + if (LibRTVecs_I64TypeObj == NULL) { return -1; } - if (!PyType_Check(I64TypeObj)) { + if (!PyType_Check(LibRTVecs_I64TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i64 is not a type"); return -1; } - I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); - if (I32TypeObj == NULL) { + LibRTVecs_I32TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i32"); + if (LibRTVecs_I32TypeObj == NULL) { return -1; } - if (!PyType_Check(I32TypeObj)) { + if (!PyType_Check(LibRTVecs_I32TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i32 is not a type"); return -1; } - I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); - if (I16TypeObj == NULL) { + LibRTVecs_I16TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "i16"); + if (LibRTVecs_I16TypeObj == NULL) { return -1; } - if (!PyType_Check(I16TypeObj)) { + if (!PyType_Check(LibRTVecs_I16TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.i16 is not a type"); return -1; } - U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); - if (U8TypeObj == NULL) { + LibRTVecs_U8TypeObj = (PyTypeObject *)PyObject_GetAttrString(ext, "u8"); + if (LibRTVecs_U8TypeObj == NULL) { return -1; } - if (!PyType_Check(U8TypeObj)) { + if (!PyType_Check(LibRTVecs_U8TypeObj)) { PyErr_SetString(PyExc_TypeError, "mypy_extensions.u8 is not a type"); return -1; } diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index eaf0271735cbf..484d6dd607b97 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -480,10 +480,10 @@ extern PyTypeObject VecTType; extern PyTypeObject VecNestedType; // Type objects correponding the 'i64', 'i32', 'i16, and 'u8' types -extern PyTypeObject *I64TypeObj; -extern PyTypeObject *I32TypeObj; -extern PyTypeObject *I16TypeObj; -extern PyTypeObject *U8TypeObj; +extern PyTypeObject *LibRTVecs_I64TypeObj; +extern PyTypeObject *LibRTVecs_I32TypeObj; +extern PyTypeObject *LibRTVecs_I16TypeObj; +extern PyTypeObject *LibRTVecs_U8TypeObj; extern VecI64Features I64Features; extern VecI32Features I32Features; From dcd314f92a615b66cd0a8e029eaa6af05f17c91d Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 13:49:32 +0000 Subject: [PATCH 196/208] Rename *Features to *API, also add missing name prefixes --- mypyc/lib-rt/vecs/librt_vecs.c | 16 ++++---- mypyc/lib-rt/vecs/librt_vecs.h | 66 ++++++++++++++++---------------- mypyc/lib-rt/vecs/vec_bool.c | 2 +- mypyc/lib-rt/vecs/vec_float.c | 2 +- mypyc/lib-rt/vecs/vec_i16.c | 2 +- mypyc/lib-rt/vecs/vec_i32.c | 2 +- mypyc/lib-rt/vecs/vec_i64.c | 2 +- mypyc/lib-rt/vecs/vec_nested.c | 2 +- mypyc/lib-rt/vecs/vec_t.c | 2 +- mypyc/lib-rt/vecs/vec_template.c | 2 +- mypyc/lib-rt/vecs/vec_u8.c | 2 +- 11 files changed, 51 insertions(+), 49 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 02edee1cee01a..278ce444c6726 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -802,14 +802,14 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) } static VecCapsule Capsule = { - &TFeatures, - &NestedFeatures, - &I64Features, - &I32Features, - &I16Features, - &U8Features, - &FloatFeatures, - &BoolFeatures, + &Vec_TAPI, + &Vec_NestedAPI, + &Vec_I64API, + &Vec_I32API, + &Vec_I16API, + &Vec_U8API, + &Vec_FloatAPI, + &Vec_BoolAPI, }; #endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 484d6dd607b97..654470442d8fa 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -264,7 +264,7 @@ typedef tuple_T2VFF VecFloatPopResult; typedef tuple_T2VCC VecBoolPopResult; // vec[i64] operations + type objects (stored in a capsule) -typedef struct _VecI64Features { +typedef struct _VecI64API { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecI64 (*alloc)(Py_ssize_t, Py_ssize_t); @@ -280,10 +280,10 @@ typedef struct _VecI64Features { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int64_t); // iter? -} VecI64Features; +} VecI64API; // vec[i32] operations + type objects (stored in a capsule) -typedef struct _VecI32Features { +typedef struct _VecI32API { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecI32 (*alloc)(Py_ssize_t, Py_ssize_t); @@ -299,10 +299,10 @@ typedef struct _VecI32Features { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int32_t); // iter? -} VecI32Features; +} VecI32API; // vec[i16] operations + type objects (stored in a capsule) -typedef struct _VecI16Features { +typedef struct _VecI16API { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecI16 (*alloc)(Py_ssize_t, Py_ssize_t); @@ -318,10 +318,10 @@ typedef struct _VecI16Features { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, int16_t); // iter? -} VecI16Features; +} VecI16API; // vec[u8] operations + type objects (stored in a capsule) -typedef struct _VecU8Features { +typedef struct _VecU8API { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecU8 (*alloc)(Py_ssize_t, Py_ssize_t); @@ -337,10 +337,10 @@ typedef struct _VecU8Features { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, uint8_t); // iter? -} VecU8Features; +} VecU8API; // vec[float] operations + type objects (stored in a capsule) -typedef struct _VecFloatFeatures { +typedef struct _VecFloatAPI { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecFloat (*alloc)(Py_ssize_t, Py_ssize_t); @@ -356,10 +356,10 @@ typedef struct _VecFloatFeatures { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, double); // iter? -} VecFloatFeatures; +} VecFloatAPI; // vec[bool] operations + type objects (stored in a capsule) -typedef struct _VecBoolFeatures { +typedef struct _VecBoolAPI { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecBool (*alloc)(Py_ssize_t, Py_ssize_t); @@ -375,7 +375,7 @@ typedef struct _VecBoolFeatures { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, char); // iter? -} VecBoolFeatures; +} VecBoolAPI; #ifndef MYPYC_DECLARED_tuple_T2VOO #define MYPYC_DECLARED_tuple_T2VOO @@ -391,7 +391,7 @@ typedef tuple_T2VOO VecTPopResult; // vec[T] operations + type objects (stored in a capsule) // // T is a class type or class type | None -typedef struct _VecTFeatures { +typedef struct _VecTAPI { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecT (*alloc)(Py_ssize_t, Py_ssize_t, size_t); @@ -407,7 +407,7 @@ typedef struct _VecTFeatures { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); // iter? -} VecTFeatures; +} VecTAPI; #ifndef MYPYC_DECLARED_tuple_T2VvVi @@ -422,7 +422,7 @@ static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; typedef tuple_T2VvVi VecNestedPopResult; // Nested vec operations + type objects (stored in a capsule) -typedef struct _VecNestedFeatures { +typedef struct _VecNestedAPI { PyTypeObject *boxed_type; PyTypeObject *buf_type; VecNested (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); @@ -438,17 +438,17 @@ typedef struct _VecNestedFeatures { // PyObject *(*concat)(PyObject *, PyObject *); // bool (*contains)(PyObject *, PyObject *); // iter? -} VecNestedFeatures; +} VecNestedAPI; typedef struct { - VecTFeatures *t; - VecNestedFeatures *nested; - VecI64Features *i64; - VecI32Features *i32; - VecI16Features *i16; - VecU8Features *u8; - VecFloatFeatures *float_; - VecBoolFeatures *bool_; + VecTAPI *t; + VecNestedAPI *nested; + VecI64API *i64; + VecI32API *i32; + VecI16API *i16; + VecU8API *u8; + VecFloatAPI *float_; + VecBoolAPI *bool_; } VecCapsule; #define VEC_BUF_SIZE(b) ((b)->ob_base.ob_size) @@ -461,6 +461,7 @@ typedef struct { // Type objects +// Buffer type objects that store vec items extern PyTypeObject VecI64BufType; extern PyTypeObject VecI32BufType; extern PyTypeObject VecI16BufType; @@ -470,6 +471,7 @@ extern PyTypeObject VecBoolBufType; extern PyTypeObject VecTBufType; extern PyTypeObject VecNestedBufType; +// Wrapper type objects for boxed vec values extern PyTypeObject VecI64Type; extern PyTypeObject VecI32Type; extern PyTypeObject VecI16Type; @@ -485,14 +487,14 @@ extern PyTypeObject *LibRTVecs_I32TypeObj; extern PyTypeObject *LibRTVecs_I16TypeObj; extern PyTypeObject *LibRTVecs_U8TypeObj; -extern VecI64Features I64Features; -extern VecI32Features I32Features; -extern VecI16Features I16Features; -extern VecU8Features U8Features; -extern VecFloatFeatures FloatFeatures; -extern VecBoolFeatures BoolFeatures; -extern VecTFeatures TFeatures; -extern VecNestedFeatures NestedFeatures; +extern VecI64API Vec_I64API; +extern VecI32API Vec_I32API; +extern VecI16API Vec_I16API; +extern VecU8API Vec_U8API; +extern VecFloatAPI Vec_FloatAPI; +extern VecBoolAPI Vec_BoolAPI; +extern VecTAPI Vec_TAPI; +extern VecNestedAPI Vec_NestedAPI; static inline int Vec_CheckFloatError(PyObject *o) { if (PyFloat_Check(o)) { diff --git a/mypyc/lib-rt/vecs/vec_bool.c b/mypyc/lib-rt/vecs/vec_bool.c index 197fd8d9d5417..ea324164779c9 100644 --- a/mypyc/lib-rt/vecs/vec_bool.c +++ b/mypyc/lib-rt/vecs/vec_bool.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "bool" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_BOOL #define ITEM_C_TYPE char -#define FEATURES BoolFeatures +#define FEATURES Vec_BoolAPI #define BOX_ITEM VecBool_BoxItem #define UNBOX_ITEM VecBool_UnboxItem diff --git a/mypyc/lib-rt/vecs/vec_float.c b/mypyc/lib-rt/vecs/vec_float.c index efc02cb189bbc..3952c01e711cb 100644 --- a/mypyc/lib-rt/vecs/vec_float.c +++ b/mypyc/lib-rt/vecs/vec_float.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "float" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_FLOAT #define ITEM_C_TYPE double -#define FEATURES FloatFeatures +#define FEATURES Vec_FloatAPI #define BOX_ITEM VecFloat_BoxItem #define UNBOX_ITEM VecFloat_UnboxItem diff --git a/mypyc/lib-rt/vecs/vec_i16.c b/mypyc/lib-rt/vecs/vec_i16.c index 658208505f48f..85a0055225d56 100644 --- a/mypyc/lib-rt/vecs/vec_i16.c +++ b/mypyc/lib-rt/vecs/vec_i16.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "i16" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I16 #define ITEM_C_TYPE int16_t -#define FEATURES I16Features +#define FEATURES Vec_I16API #define BOX_ITEM VecI16_BoxItem #define UNBOX_ITEM VecI16_UnboxItem diff --git a/mypyc/lib-rt/vecs/vec_i32.c b/mypyc/lib-rt/vecs/vec_i32.c index d8efef0b3d579..9eb595994483a 100644 --- a/mypyc/lib-rt/vecs/vec_i32.c +++ b/mypyc/lib-rt/vecs/vec_i32.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "i32" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I32 #define ITEM_C_TYPE int32_t -#define FEATURES I32Features +#define FEATURES Vec_I32API #define BOX_ITEM VecI32_BoxItem #define UNBOX_ITEM VecI32_UnboxItem diff --git a/mypyc/lib-rt/vecs/vec_i64.c b/mypyc/lib-rt/vecs/vec_i64.c index d96676688f491..e6b7df693333f 100644 --- a/mypyc/lib-rt/vecs/vec_i64.c +++ b/mypyc/lib-rt/vecs/vec_i64.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "i64" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_I64 #define ITEM_C_TYPE int64_t -#define FEATURES I64Features +#define FEATURES Vec_I64API #define BOX_ITEM VecI64_BoxItem #define UNBOX_ITEM VecI64_UnboxItem diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c index 569e038f010cd..caed937dff9b8 100644 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ b/mypyc/lib-rt/vecs/vec_nested.c @@ -492,7 +492,7 @@ PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *itera return VecNested_Box(v); } -VecNestedFeatures NestedFeatures = { +VecNestedAPI Vec_NestedAPI = { &VecNestedType, &VecNestedBufType, VecNested_New, diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c index e21089ff700b3..342143492f26e 100644 --- a/mypyc/lib-rt/vecs/vec_t.c +++ b/mypyc/lib-rt/vecs/vec_t.c @@ -486,7 +486,7 @@ PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable) { return VecT_Box(v, item_type); } -VecTFeatures TFeatures = { +VecTAPI Vec_TAPI = { &VecTType, &VecTBufType, VecT_New, diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 4292e3ffb66d4..b490c4c1d6fa0 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -368,7 +368,7 @@ PyTypeObject VEC_TYPE = { .tp_methods = vec_methods, }; -NAME(Features) FEATURES = { +NAME(API) FEATURES = { &VEC_TYPE, &BUF_TYPE, FUNC(New), diff --git a/mypyc/lib-rt/vecs/vec_u8.c b/mypyc/lib-rt/vecs/vec_u8.c index 2060f423413c4..1d59f177c3104 100644 --- a/mypyc/lib-rt/vecs/vec_u8.c +++ b/mypyc/lib-rt/vecs/vec_u8.c @@ -9,7 +9,7 @@ #define ITEM_TYPE_STR "u8" #define ITEM_TYPE_MAGIC VEC_ITEM_TYPE_U8 #define ITEM_C_TYPE uint8_t -#define FEATURES U8Features +#define FEATURES Vec_U8API #define BOX_ITEM VecU8_BoxItem #define UNBOX_ITEM VecU8_UnboxItem From a92651d0e3e7a803fa141484fe03e69b8b2545a3 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Mon, 26 Jan 2026 14:45:04 +0000 Subject: [PATCH 197/208] Add documentation in comments --- mypyc/lib-rt/vecs/librt_vecs.c | 69 ++++++++++++++++++++++++++++++++++ mypyc/lib-rt/vecs/librt_vecs.h | 3 ++ 2 files changed, 72 insertions(+) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 278ce444c6726..c6b09724cbff1 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -1,3 +1,72 @@ +// Definition of the mypyc.vecs module, which implements 'vec' and related functionality. +// +// NOTE: This is still experimental and work in progress. +// +// vec[t] is a sequence type that is optimized for use in mypyc-compiled code. +// Performance in interpreted code is a secondary concern. It's primarily optimized +// to provide fast item get/set operations and fast len(), especially for primitive +// value item types, such as 'i32', 'float' or 'bool'. +// +// Example: +// +// from mypy_extensions import i64 +// +// v = vec[i64]() +// v = append(v, 123) +// print(v[0]) +// v[-1] = 234 +// +// Key properties: +// +// * In native code, a vec instance is an immutable value type with two fields +// (length, item buffer). +// * The item buffer can only store types of a specific type (enforced at runtime). +// * There are specialized item buffer types for certain primitive value item types, +// such as 'i32', 'float' and 'bool'. These use a packed value encoding. +// * Since vec values are immutable (only the buffer is mutable), any operation +// that changes the length (such as append) must return the updated vec value. +// * Vec values can be boxed to PyObject *; this happens transparently and uses +// a wrapper object. +// * The item type can be a class type, or 't | None' if t is a class type. However, +// optional value item types such as 'float | None' are not supported, and 'int' +// is not valid as item type -- a native integer type such as `i64` or `u8` must +// be used instead. +// * Nested vecs are also supported (e.g. vec[vec[i32]]). +// * All features are also available in interpreted code, but often at some +// performance cost compared to using plain lists. +// +// Implementation summary: +// +// * In a non-native context, instances are constructed via a temporary VecGenericAlias +// object, which is the result of 'vec[]' (indexing the 'vec' type). +// * The module exports a C API via a capsule that mypyc compiled code uses for most +// operations. +// * There are multiple C structs that represent vec value objects with different item +// types, such as VecI64 and VecT. The latter is for arbitrary reference objects +// ('PyObject *' items internally). +// * There are also multiple buffer types, such as VecI64Buf and VecTBuf. +// * Similarly, there are multiple Python wrapper object types for boxed vecs, such as +// VecI64Type and VecTType. +// * An empty vec in compiled code can use a NULL pointer to represent the buffer, +// saving an object allocation in a common use case. +// +// Motivation: +// * A vec with value type items, such as vec[i64] or vec[vec[i32]], is very efficient +// compared to a list object, since values are not boxed, there is no need for runtime +// type checks on read, and index overflow checks are very fast due to the immutable +// value encoding. +// * A vec with a reference item type, such as vec[str], is often faster than a list, +// since there is no need for a cast on item read and the index checks are faster +// (among other things), but here the benefit is much smaller compared to using a +// value item type. +// +// Summary of files: +// * vec_t.c implements vec[t] and vec[t | None] for reference types +// * vec_nested.c implements nested vecs, such as vec[vec[str]] +// * vec_template.c is #included in multiples files (e.g. vec_i64.c) to produce +// specialized code for vecs with value item types (e.g. vec[i64]) +// * This file defines the C extension and has some shared code + #define PY_SSIZE_T_CLEAN #include #include "librt_vecs.h" diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 654470442d8fa..bd89fb4866d4b 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -1,6 +1,9 @@ #ifndef VEC_H_INCL #define VEC_H_INCL +// Header for the implementation of librt.vecs, which defines the 'vec' type. +// Refer to librt_vecs.c for more detailed information. + #ifndef MYPYC_EXPERIMENTAL static int From 8b3f8fff773322ad3d5fbf635a0b1fb927bdf30e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 11:00:58 +0000 Subject: [PATCH 198/208] Lint --- mypyc/build.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mypyc/build.py b/mypyc/build.py index 2f2709ae6e55f..a7f43e1d6d2e1 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -117,10 +117,7 @@ class ModDesc(NamedTuple): "vecs/vec_t.c", "vecs/vec_nested.c", ], - [ - "vecs/librt_vecs.h", - "vecs/vec_template.c", - ], + ["vecs/librt_vecs.h", "vecs/vec_template.c"], ["vecs"], ), ] From 617ff8e54c6a63b3513aea386d8ae9743a066ee4 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 11:24:36 +0000 Subject: [PATCH 199/208] Update comment --- mypyc/lib-rt/vecs/librt_vecs.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index c6b09724cbff1..52394196bf83a 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -25,6 +25,9 @@ // such as 'i32', 'float' and 'bool'. These use a packed value encoding. // * Since vec values are immutable (only the buffer is mutable), any operation // that changes the length (such as append) must return the updated vec value. +// * The buffer often a has higher capacity than the length of a vec. Append +// operations only allocate a new buffer if the capacity is exhausted, for O(1) +// amortized time complexity for appends. // * Vec values can be boxed to PyObject *; this happens transparently and uses // a wrapper object. // * The item type can be a class type, or 't | None' if t is a class type. However, From c8db19a12cedac631f2366444253b3f6a4a11fdf Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 11:03:24 +0000 Subject: [PATCH 200/208] Remove vec[t] and nested vec support temporarily --- mypyc/build.py | 2 - mypyc/lib-rt/setup.py | 2 - mypyc/lib-rt/vecs/librt_vecs.c | 112 +---- mypyc/lib-rt/vecs/librt_vecs.h | 203 -------- mypyc/lib-rt/vecs/vec_nested.c | 508 -------------------- mypyc/lib-rt/vecs/vec_t.c | 502 ------------------- mypyc/test-data/run-vecs-i64-interp.test | 2 - mypyc/test-data/run-vecs-nested-interp.test | 317 ------------ mypyc/test-data/run-vecs-t-interp.test | 457 ------------------ mypyc/test/test_run.py | 2 - 10 files changed, 3 insertions(+), 2104 deletions(-) delete mode 100644 mypyc/lib-rt/vecs/vec_nested.c delete mode 100644 mypyc/lib-rt/vecs/vec_t.c delete mode 100644 mypyc/test-data/run-vecs-nested-interp.test delete mode 100644 mypyc/test-data/run-vecs-t-interp.test diff --git a/mypyc/build.py b/mypyc/build.py index a7f43e1d6d2e1..c2c38eeaa269e 100644 --- a/mypyc/build.py +++ b/mypyc/build.py @@ -114,8 +114,6 @@ class ModDesc(NamedTuple): "vecs/vec_u8.c", "vecs/vec_float.c", "vecs/vec_bool.c", - "vecs/vec_t.c", - "vecs/vec_nested.c", ], ["vecs/librt_vecs.h", "vecs/vec_template.c"], ["vecs"], diff --git a/mypyc/lib-rt/setup.py b/mypyc/lib-rt/setup.py index ee2cc3bcdd93c..9b477e457d758 100644 --- a/mypyc/lib-rt/setup.py +++ b/mypyc/lib-rt/setup.py @@ -142,8 +142,6 @@ def run(self) -> None: "vecs/vec_u8.c", "vecs/vec_float.c", "vecs/vec_bool.c", - "vecs/vec_t.c", - "vecs/vec_nested.c", ], include_dirs=[".", "vecs"], extra_compile_args=cflags, diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 52394196bf83a..77c9837a1352f 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -96,31 +96,9 @@ typedef struct { static PyObject *vec_generic_alias_call(PyObject *self, PyObject *args, PyObject *kw) { - static char *kwlist[] = {"", NULL}; - PyObject *init = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kw, "|O:vec", kwlist, &init)) { - return NULL; - } - VecGenericAlias *p = (VecGenericAlias *)self; - if (p->depth == 0) { - if (init == NULL) { - VecT vec = VecT_New(0, 0, p->item_type); - if (VEC_IS_ERROR(vec)) - return NULL; - return VecT_Box(vec, p->item_type); - } else { - return VecT_FromIterable(p->item_type, init); - } - } else { - if (init == NULL) { - VecNested vec = VecNested_New(0, 0, p->item_type, p->depth); - if (VEC_IS_ERROR(vec)) - return NULL; - return VecNested_Box(vec); - } else { - return VecNested_FromIterable(p->item_type, p->depth, init); - } - } + PyErr_SetString(PyExc_NotImplementedError, + "non-specialized vec construction not yet supported"); + return NULL; } static int @@ -619,26 +597,6 @@ static PyObject *vec_append(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecBool_Box(v); - } else if (VecT_Check(vec)) { - VecT v = ((VecTObject *)vec)->vec; - if (!VecT_ItemCheck(v, item, v.buf->item_type)) { - return NULL; - } - VEC_INCREF(v); - v = VecT_Append(v, item, v.buf->item_type); - if (VEC_IS_ERROR(v)) - return NULL; - return VecT_Box(v, v.buf->item_type); - } else if (VecNested_Check(vec)) { - VecNested v = ((VecNestedObject *)vec)->vec; - VecNestedBufItem vecitem; - if (VecNested_UnboxItem(v, item, &vecitem) < 0) - return NULL; - VEC_INCREF(v); - v = VecNested_Append(v, vecitem); - if (VEC_IS_ERROR(v)) - return NULL; - return VecNested_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", Py_TYPE(vec)->tp_name); @@ -720,26 +678,6 @@ static PyObject *vec_remove(PyObject *self, PyObject *args) if (VEC_IS_ERROR(v)) return NULL; return VecBool_Box(v); - } else if (VecT_Check(vec)) { - VecT v = ((VecTObject *)vec)->vec; - if (!VecT_ItemCheck(v, item, v.buf->item_type)) { - return NULL; - } - VEC_INCREF(v); - v = VecT_Remove(v, item); - if (VEC_IS_ERROR(v)) - return NULL; - return VecT_Box(v, v.buf->item_type); - } else if (VecNested_Check(vec)) { - VecNested v = ((VecNestedObject *)vec)->vec; - VecNestedBufItem vecitem; - if (VecNested_UnboxItem(v, item, &vecitem) < 0) - return NULL; - VEC_INCREF(v); - v = VecNested_Remove(v, vecitem); - if (VEC_IS_ERROR(v)) - return NULL; - return VecNested_Box(v); } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", Py_TYPE(vec)->tp_name); @@ -818,38 +756,6 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) if ((result_item0 = VecBool_Box(r.f0)) == NULL) return NULL; result_item1 = VecBool_BoxItem(r.f1); - } else if (VecT_Check(vec)) { - VecT v = ((VecTObject *)vec)->vec; - VEC_INCREF(v); - VecTPopResult r; - r = VecT_Pop(v, index); - if (VEC_IS_ERROR(r.f0)) - return NULL; - result_item0 = VecT_Box(r.f0, v.buf->item_type); - if (result_item0 == NULL) { - Py_DECREF(r.f1); - return NULL; - } - result_item1 = r.f1; - } else if (VecNested_Check(vec)) { - VecNested v = ((VecNestedObject *)vec)->vec; - VEC_INCREF(v); - VecNestedPopResult r; - r = VecNested_Pop(v, index); - if (VEC_IS_ERROR(r.f0)) - return NULL; - result_item0 = VecNested_Box(r.f0); - if (result_item0 == NULL) { - Py_DECREF(r.f0.buf); - Py_DECREF(r.f1.buf); - return NULL; - } - result_item1 = VecNested_BoxItem(r.f0, r.f1); - if (result_item1 == NULL) { - Py_DECREF(result_item0); - Py_DECREF(r.f1.buf); - return NULL; - } } else { PyErr_Format(PyExc_TypeError, "vec argument expected, got %.100s", Py_TYPE(vec)->tp_name); @@ -874,8 +780,6 @@ static PyObject *vec_pop(PyObject *self, PyObject *args) } static VecCapsule Capsule = { - &Vec_TAPI, - &Vec_NestedAPI, &Vec_I64API, &Vec_I32API, &Vec_I16API, @@ -942,16 +846,6 @@ librt_vecs_module_exec(PyObject *m) if (PyType_Ready(&VecGenericAliasType) < 0) return -1; - if (PyType_Ready(&VecTType) < 0) - return -1; - if (PyType_Ready(&VecTBufType) < 0) - return -1; - - if (PyType_Ready(&VecNestedType) < 0) - return -1; - if (PyType_Ready(&VecNestedBufType) < 0) - return -1; - if (PyType_Ready(&VecI64Type) < 0) return -1; if (PyType_Ready(&VecI64BufType) < 0) diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index bd89fb4866d4b..7a9ecb84b2c85 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -380,72 +380,7 @@ typedef struct _VecBoolAPI { // iter? } VecBoolAPI; -#ifndef MYPYC_DECLARED_tuple_T2VOO -#define MYPYC_DECLARED_tuple_T2VOO -typedef struct tuple_T2VOO { - VecT f0; - PyObject *f1; -} tuple_T2VOO; -static tuple_T2VOO tuple_undefined_T2VOO = { { -1, NULL } , NULL }; -#endif - -typedef tuple_T2VOO VecTPopResult; - -// vec[T] operations + type objects (stored in a capsule) -// -// T is a class type or class type | None -typedef struct _VecTAPI { - PyTypeObject *boxed_type; - PyTypeObject *buf_type; - VecT (*alloc)(Py_ssize_t, Py_ssize_t, size_t); - PyObject *(*box)(VecT, size_t); - VecT (*unbox)(PyObject *, size_t); - VecT (*convert_from_nested)(VecNestedBufItem); - VecT (*append)(VecT, PyObject *, size_t); - VecTPopResult (*pop)(VecT, Py_ssize_t); - VecT (*remove)(VecT, PyObject *); - // TODO: Py_ssize_t - VecT (*slice)(VecT, int64_t, int64_t); - // PyObject *(*extend)(PyObject *, PyObject *); - // PyObject *(*concat)(PyObject *, PyObject *); - // bool (*contains)(PyObject *, PyObject *); - // iter? -} VecTAPI; - - -#ifndef MYPYC_DECLARED_tuple_T2VvVi -#define MYPYC_DECLARED_tuple_T2VvVi -typedef struct tuple_T2VvVi { - VecNested f0; - VecNestedBufItem f1; -} tuple_T2VvVi; -static tuple_T2VvVi tuple_undefined_T2VvVi = { { -1, NULL } , { -1, NULL } }; -#endif - -typedef tuple_T2VvVi VecNestedPopResult; - -// Nested vec operations + type objects (stored in a capsule) -typedef struct _VecNestedAPI { - PyTypeObject *boxed_type; - PyTypeObject *buf_type; - VecNested (*alloc)(Py_ssize_t, Py_ssize_t, size_t, size_t depth); - PyObject *(*box)(VecNested); - VecNested (*unbox)(PyObject *, size_t, size_t depth); - VecNested (*convert_from_nested)(VecNestedBufItem); - VecNested (*append)(VecNested, VecNestedBufItem); - VecNestedPopResult (*pop)(VecNested, Py_ssize_t); - VecNested (*remove)(VecNested, VecNestedBufItem); - // TODO: Py_ssize_t - VecNested (*slice)(VecNested, int64_t, int64_t); - // PyObject *(*extend)(PyObject *, PyObject *); - // PyObject *(*concat)(PyObject *, PyObject *); - // bool (*contains)(PyObject *, PyObject *); - // iter? -} VecNestedAPI; - typedef struct { - VecTAPI *t; - VecNestedAPI *nested; VecI64API *i64; VecI32API *i32; VecI16API *i16; @@ -471,8 +406,6 @@ extern PyTypeObject VecI16BufType; extern PyTypeObject VecU8BufType; extern PyTypeObject VecFloatBufType; extern PyTypeObject VecBoolBufType; -extern PyTypeObject VecTBufType; -extern PyTypeObject VecNestedBufType; // Wrapper type objects for boxed vec values extern PyTypeObject VecI64Type; @@ -481,8 +414,6 @@ extern PyTypeObject VecI16Type; extern PyTypeObject VecU8Type; extern PyTypeObject VecFloatType; extern PyTypeObject VecBoolType; -extern PyTypeObject VecTType; -extern PyTypeObject VecNestedType; // Type objects correponding the 'i64', 'i32', 'i16, and 'u8' types extern PyTypeObject *LibRTVecs_I64TypeObj; @@ -496,8 +427,6 @@ extern VecI16API Vec_I16API; extern VecU8API Vec_U8API; extern VecFloatAPI Vec_FloatAPI; extern VecBoolAPI Vec_BoolAPI; -extern VecTAPI Vec_TAPI; -extern VecNestedAPI Vec_NestedAPI; static inline int Vec_CheckFloatError(PyObject *o) { if (PyFloat_Check(o)) { @@ -684,138 +613,6 @@ VecBool VecBool_Append(VecBool, char x); VecBool VecBool_Remove(VecBool, char x); VecBoolPopResult VecBool_Pop(VecBool v, Py_ssize_t index); -// vec[t] operations - -static inline int VecT_Check(PyObject *o) { - return o->ob_type == &VecTType; -} - -static inline int VecT_ItemCheck(VecT v, PyObject *item, size_t item_type) { - if (PyObject_TypeCheck(item, VEC_ITEM_TYPE(item_type))) { - return 1; - } else if ((item_type & 1) && item == Py_None) { - return 1; - } else { - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); - return 0; - } -} - -VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type); -PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable); -PyObject *VecT_Box(VecT vec, size_t item_type); -VecT VecT_Append(VecT vec, PyObject *x, size_t item_type); -VecT VecT_Remove(VecT vec, PyObject *x); -VecTPopResult VecT_Pop(VecT v, Py_ssize_t index); - -// Nested vec operations - -static inline int VecNested_Check(PyObject *o) { - return o->ob_type == &VecNestedType; -} - -VecNested VecNested_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth); -PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *iterable); -PyObject *VecNested_Box(VecNested); -VecNested VecNested_Append(VecNested vec, VecNestedBufItem x); -VecNested VecNested_Remove(VecNested vec, VecNestedBufItem x); -VecNestedPopResult VecNested_Pop(VecNested v, Py_ssize_t index); - -// Return 0 on success, -1 on error. Store unboxed item in *unboxed if successful. -// Return a *borrowed* reference. -static inline int VecNested_UnboxItem(VecNested v, PyObject *item, VecNestedBufItem *unboxed) { - size_t depth = v.buf->depth; - if (depth == 1) { - if (item->ob_type == &VecTType) { - VecNestedObject *o = (VecNestedObject *)item; - if (o->vec.buf->item_type == v.buf->item_type) { - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } - } else if (item->ob_type == &VecI64Type && v.buf->item_type == VEC_ITEM_TYPE_I64) { - VecI64Object *o = (VecI64Object *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } else if (item->ob_type == &VecU8Type && v.buf->item_type == VEC_ITEM_TYPE_U8) { - VecU8Object *o = (VecU8Object *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } else if (item->ob_type == &VecFloatType && v.buf->item_type == VEC_ITEM_TYPE_FLOAT) { - VecFloatObject *o = (VecFloatObject *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } else if (item->ob_type == &VecI32Type && v.buf->item_type == VEC_ITEM_TYPE_I32) { - VecI32Object *o = (VecI32Object *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } else if (item->ob_type == &VecI16Type && v.buf->item_type == VEC_ITEM_TYPE_I16) { - VecI16Object *o = (VecI16Object *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } else if (item->ob_type == &VecBoolType && v.buf->item_type == VEC_ITEM_TYPE_BOOL) { - VecBoolObject *o = (VecBoolObject *)item; - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } - } else if (item->ob_type == &VecNestedType) { - VecNestedObject *o = (VecNestedObject *)item; - if (o->vec.buf->depth == v.buf->depth - 1 - && o->vec.buf->item_type == v.buf->item_type) { - unboxed->len = o->vec.len; - unboxed->buf = (PyObject *)o->vec.buf; - return 0; - } - } - // TODO: better error message - PyErr_SetString(PyExc_TypeError, "invalid item type"); - return -1; -} - -static inline PyObject *VecNested_BoxItem(VecNested v, VecNestedBufItem item) { - if (item.len < 0) - Py_RETURN_NONE; - Py_XINCREF(item.buf); - if (v.buf->depth > 1) { - // Item is a nested vec - VecNested v = { .len = item.len, .buf = (VecNestedBufObject *)item.buf }; - return VecNested_Box(v); - } else { - // Item is a non-nested vec - size_t item_type = v.buf->item_type; - if (item_type == VEC_ITEM_TYPE_I64) { - VecI64 v = { .len = item.len, .buf = (VecI64BufObject *)item.buf }; - return VecI64_Box(v); - } else if (item_type == VEC_ITEM_TYPE_U8) { - VecU8 v = { .len = item.len, .buf = (VecU8BufObject *)item.buf }; - return VecU8_Box(v); - } else if (item_type == VEC_ITEM_TYPE_FLOAT) { - VecFloat v = { .len = item.len, .buf = (VecFloatBufObject *)item.buf }; - return VecFloat_Box(v); - } else if (item_type == VEC_ITEM_TYPE_I32) { - VecI32 v = { .len = item.len, .buf = (VecI32BufObject *)item.buf }; - return VecI32_Box(v); - } else if (item_type == VEC_ITEM_TYPE_I16) { - VecI16 v = { .len = item.len, .buf = (VecI16BufObject *)item.buf }; - return VecI16_Box(v); - } else if (item_type == VEC_ITEM_TYPE_BOOL) { - VecBool v = { .len = item.len, .buf = (VecBoolBufObject *)item.buf }; - return VecBool_Box(v); - } else { - // Generic vec[t] - VecT v = { .len = item.len, .buf = (VecTBufObject *)item.buf }; - return VecT_Box(v, item_type); - } - } -} - // Misc helpers PyObject *Vec_TypeToStr(size_t item_type, size_t depth); diff --git a/mypyc/lib-rt/vecs/vec_nested.c b/mypyc/lib-rt/vecs/vec_nested.c deleted file mode 100644 index caed937dff9b8..0000000000000 --- a/mypyc/lib-rt/vecs/vec_nested.c +++ /dev/null @@ -1,508 +0,0 @@ -#ifdef MYPYC_EXPERIMENTAL -// Implementation of nested vec[t], when t is a vec type. -// -// Examples of types supported: -// - vec[vec[i64]] -// - vec[vec[vec[str]]] -// - vec[vec[str | None]] - -#define PY_SSIZE_T_CLEAN -#include -#include "librt_vecs.h" - -static inline VecNested vec_error() { - VecNested v = { .len = -1 }; - return v; -} - -static inline PyObject *box_vec_item_by_index(VecNested v, Py_ssize_t index) { - return VecNested_BoxItem(v, v.buf->items[index]); -} - -// Alloc a partially initialized vec. Caller *must* initialize len and buf->items of the -// return value. -static VecNested vec_alloc(Py_ssize_t size, size_t item_type, size_t depth) { - VecNestedBufObject *buf = PyObject_GC_NewVar(VecNestedBufObject, &VecNestedBufType, size); - if (buf == NULL) - return vec_error(); - buf->item_type = item_type; - buf->depth = depth; - if (!Vec_IsMagicItemType(item_type)) - Py_INCREF(VEC_BUF_ITEM_TYPE(buf)); - VecNested res = { .buf = buf }; - PyObject_GC_Track(buf); - return res; -} - -// Box a nested vec value, stealing 'vec'. On error, decref 'vec'. -PyObject *VecNested_Box(VecNested vec) { - VecNestedObject *obj = PyObject_GC_New(VecNestedObject, &VecNestedType); - if (obj == NULL) { - VEC_DECREF(vec); - return NULL; - } - obj->vec = vec; - PyObject_GC_Track(obj); - return (PyObject *)obj; -} - -VecNested VecNested_Unbox(PyObject *obj, size_t item_type, size_t depth) { - if (obj->ob_type == &VecNestedType) { - VecNested result = ((VecNestedObject *)obj)->vec; - if (result.buf->item_type == item_type && result.buf->depth == depth) { - VEC_INCREF(result); // TODO: Should we borrow instead? - return result; - } - } - // TODO: Better error message, with name of type - PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return vec_error(); -} - -VecNested VecNested_ConvertFromNested(VecNestedBufItem item) { - return (VecNested) { item.len, (VecNestedBufObject *)item.buf }; -} - -VecNested VecNested_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type, size_t depth) { - if (cap < size) - cap = size; - VecNested vec = vec_alloc(cap, item_type, depth); - if (VEC_IS_ERROR(vec)) - return vec; - for (Py_ssize_t i = 0; i < cap; i++) { - vec.buf->items[i].len = -1; - vec.buf->items[i].buf = NULL; - } - vec.len = size; - return vec; -} - -static PyObject *vec_repr(PyObject *self) { - VecNested v = ((VecNestedObject *)self)->vec; - return Vec_GenericRepr(self, v.buf->item_type, v.buf->depth, 1); -} - -static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecNested v = ((VecNestedObject *)o)->vec; - if ((size_t)i < (size_t)v.len) { - return box_vec_item_by_index(v, i); - } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - return box_vec_item_by_index(v, i + v.len); - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } -} - -VecNested VecNested_Slice(VecNested vec, int64_t start, int64_t end) { - if (start < 0) - start += vec.len; - if (end < 0) - end += vec.len; - if (start < 0) - start = 0; - if (start >= vec.len) - start = vec.len; - if (end < start) - end = start; - if (end > vec.len) - end = vec.len; - int64_t slicelength = end - start; - VecNested res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); - if (VEC_IS_ERROR(res)) - return res; - res.len = slicelength; - for (Py_ssize_t i = 0; i < slicelength; i++) { - VecNestedBufItem item = vec.buf->items[start + i]; - Py_XINCREF(item.buf); - res.buf->items[i] = item; - } - return res; -} - -static PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecNested vec = ((VecNestedObject *)self)->vec; - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) - return NULL; - if ((size_t)i < (size_t)vec.len) { - return box_vec_item_by_index(vec, i); - } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { - return box_vec_item_by_index(vec, i + vec.len); - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } - } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) - return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecNested res = vec_alloc(slicelength, vec.buf->item_type, vec.buf->depth); - if (VEC_IS_ERROR(res)) - return NULL; - res.len = slicelength; - Py_ssize_t j = start; - for (Py_ssize_t i = 0; i < slicelength; i++) { - VecNestedBufItem item = vec.buf->items[j]; - Py_INCREF(item.buf); - res.buf->items[i] = item; - j += step; - } - return VecNested_Box(res); - } else { - PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", - item->ob_type->tp_name); - return NULL; - } -} - -static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - VecNested v = ((VecNestedObject *)self)->vec; - if ((size_t)i + (size_t)v.len < (size_t)v.len) { - i += v.len; - } - if ((size_t)i < (size_t)v.len) { - VecNestedBufItem item; - if (VecNested_UnboxItem(v, o, &item) < 0) - return -1; - VEC_INCREF(item); - VEC_DECREF(v.buf->items[i]); - v.buf->items[i] = item; - return 0; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return -1; - } -} - -static PyObject *compare_vec_eq(VecNested x, VecNested y, int op) { - int cmp = 1; - PyObject *res; - if (x.len != y.len - || x.buf->item_type != y.buf->item_type - || x.buf->depth != y.buf->depth) { - cmp = 0; - } else { - for (Py_ssize_t i = 0; i < x.len; i++) { - PyObject *x_item = box_vec_item_by_index(x, i); - PyObject *y_item = box_vec_item_by_index(y, i); - int itemcmp = PyObject_RichCompareBool(x_item, y_item, Py_EQ); - Py_DECREF(x_item); - Py_DECREF(y_item); - if (itemcmp < 0) - return NULL; - if (!itemcmp) { - cmp = 0; - break; - } - } - } - if (op == Py_NE) - cmp = cmp ^ 1; - res = cmp ? Py_True : Py_False; - Py_INCREF(res); - return res; -} - -PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { - PyObject *res; - if (op == Py_EQ || op == Py_NE) { - if (other->ob_type != &VecNestedType) { - res = op == Py_EQ ? Py_False : Py_True; - } else { - return compare_vec_eq(((VecNestedObject *)self)->vec, - ((VecNestedObject *)other)->vec, op); - } - } else - res = Py_NotImplemented; - Py_INCREF(res); - return res; -} - -// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. -VecNested VecNested_Append(VecNested vec, VecNestedBufItem x) { - Py_ssize_t cap = VEC_CAP(vec); - VEC_INCREF(x); - if (vec.len < cap) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t new_size = 2 * cap + 1; - // TODO: Avoid initializing to zero here - VecNested new = vec_alloc(new_size, vec.buf->item_type, vec.buf->depth); - if (VEC_IS_ERROR(new)) { - VEC_DECREF(x); - // The input vec is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(vec); - return new; - } - // Copy items to new vec. - memcpy(new.buf->items, vec.buf->items, sizeof(VecNestedBufItem) * vec.len); - // TODO: How to safely represent deleted items? - memset(new.buf->items + vec.len, 0, sizeof(VecNestedBufItem) * (new_size - vec.len)); - // Clear the items in the old vec. We avoid reference count manipulation. - memset(vec.buf->items, 0, sizeof(VecNestedBufItem) * vec.len); - new.buf->items[vec.len] = x; - new.len = vec.len + 1; - VEC_DECREF(vec); - return new; - } -} - -// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. -VecNested VecNested_Remove(VecNested self, VecNestedBufItem arg) { - VecNestedBufItem *items = self.buf->items; - - PyObject *boxed_arg = VecNested_BoxItem(self, arg); - if (boxed_arg == NULL) { - // The input self is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(self); - return vec_error(); - } - - for (Py_ssize_t i = 0; i < self.len; i++) { - int match = 0; - if (items[i].len == arg.len && items[i].buf == arg.buf) - match = 1; - else { - PyObject *item = box_vec_item_by_index(self, i); - if (item == NULL) { - Py_DECREF(boxed_arg); - // The input self is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(self); - return vec_error(); - } - int itemcmp = PyObject_RichCompareBool(item, boxed_arg, Py_EQ); - Py_DECREF(item); - if (itemcmp < 0) { - Py_DECREF(boxed_arg); - // The input self is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(self); - return vec_error(); - } - match = itemcmp; - } - if (match) { - if (i < self.len - 1) { - Py_CLEAR(items[i].buf); - for (; i < self.len - 1; i++) { - items[i] = items[i + 1]; - } - Py_XINCREF(items[self.len - 1].buf); - } - self.len--; - Py_DECREF(boxed_arg); - // Return the stolen reference without INCREF - return self; - } - } - Py_DECREF(boxed_arg); - PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - // The input self is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(self); - return vec_error(); -} - -// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. -VecNestedPopResult VecNested_Pop(VecNested v, Py_ssize_t index) { - VecNestedPopResult result; - - if (index < 0) - index += v.len; - - if (index < 0 || index >= v.len) { - PyErr_SetString(PyExc_IndexError, "index out of range"); - // The input v is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(v); - result.f0 = vec_error(); - result.f1.len = 0; - result.f1.buf = NULL; - return result; - } - - VecNestedBufItem *items = v.buf->items; - result.f1 = items[index]; - for (Py_ssize_t i = index; i < v.len - 1; i++) - items[i] = items[i + 1]; - if (v.len > 0) - Py_XINCREF(items[v.len - 1].buf); - v.len--; - // Return the stolen reference without INCREF - result.f0 = v; - return result; -} - -static int -VecNested_traverse(VecNestedObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->vec.buf); - return 0; -} - -static int -VecNested_clear(VecNestedObject *self) -{ - Py_CLEAR(self->vec.buf); - return 0; -} - -static void -VecNested_dealloc(VecNestedObject *self) -{ - PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecNested_dealloc) - Py_CLEAR(self->vec.buf); - Py_TYPE(self)->tp_free((PyObject *)self); - Py_TRASHCAN_END -} - -static int -VecNestedBuf_traverse(VecNestedBufObject *self, visitproc visit, void *arg) -{ - if (!Vec_IsMagicItemType(self->item_type)) - Py_VISIT(VEC_BUF_ITEM_TYPE(self)); - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_VISIT(self->items[i].buf); - } - return 0; -} - -static int -VecNestedBuf_clear(VecNestedBufObject *self) -{ - if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { - Py_DECREF(VEC_BUF_ITEM_TYPE(self)); - self->item_type = 0; - } - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_CLEAR(self->items[i].buf); - } - return 0; -} - -static void -VecNestedBuf_dealloc(VecNestedBufObject *self) -{ - PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecNestedBuf_dealloc) - if (self->item_type && !Vec_IsMagicItemType(self->item_type)) { - Py_DECREF(VEC_BUF_ITEM_TYPE(self)); - self->item_type = 0; - } - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_CLEAR(self->items[i].buf); - } - Py_TYPE(self)->tp_free((PyObject *)self); - Py_TRASHCAN_END -} - -static Py_ssize_t vec_ext_length(PyObject *o) { - return ((VecNestedObject *)o)->vec.len; -} - -static PyMappingMethods VecNestedMapping = { - .mp_length = vec_ext_length, - .mp_subscript = vec_subscript, -}; - -static PySequenceMethods VecNestedSequence = { - .sq_item = vec_get_item, - .sq_ass_item = vec_ass_item, -}; - -static PyMethodDef vec_methods[] = { - {NULL, NULL, 0, NULL}, /* Sentinel */ -}; - -PyTypeObject VecNestedBufType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecbuf", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecNestedBufObject) - sizeof(VecNestedBufItem), - .tp_itemsize = sizeof(VecNestedBufItem), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecNestedBuf_traverse, - //.tp_new = vecbuf_i64_new, //?? - .tp_free = PyObject_GC_Del, - .tp_clear = (inquiry)VecNestedBuf_clear, - .tp_dealloc = (destructor)VecNestedBuf_dealloc, -}; - -PyTypeObject VecNestedType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecNestedObject) - sizeof(PyObject *), - .tp_itemsize = sizeof(PyObject *), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecNested_traverse, - .tp_clear = (inquiry)VecNested_clear, - .tp_dealloc = (destructor)VecNested_dealloc, - //.tp_free = PyObject_GC_Del, - .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecNestedSequence, - .tp_as_mapping = &VecNestedMapping, - .tp_richcompare = vec_richcompare, - .tp_methods = vec_methods, - // TODO: free -}; - -PyObject *VecNested_FromIterable(size_t item_type, size_t depth, PyObject *iterable) { - VecNested v = vec_alloc(0, item_type, depth); - if (VEC_IS_ERROR(v)) - return NULL; - v.len = 0; - - PyObject *iter = PyObject_GetIter(iterable); - if (iter == NULL) { - VEC_DECREF(v); - return NULL; - } - PyObject *item; - while ((item = PyIter_Next(iter)) != NULL) { - VecNestedBufItem vecitem; - if (VecNested_UnboxItem(v, item, &vecitem) < 0) { - Py_DECREF(iter); - VEC_DECREF(v); - Py_DECREF(item); - return NULL; - } - v = VecNested_Append(v, vecitem); - Py_DECREF(item); - if (VEC_IS_ERROR(v)) { - Py_DECREF(iter); - VEC_DECREF(v); - return NULL; - } - } - Py_DECREF(iter); - if (PyErr_Occurred()) { - VEC_DECREF(v); - return NULL; - } - return VecNested_Box(v); -} - -VecNestedAPI Vec_NestedAPI = { - &VecNestedType, - &VecNestedBufType, - VecNested_New, - VecNested_Box, - VecNested_Unbox, - VecNested_ConvertFromNested, - VecNested_Append, - VecNested_Pop, - VecNested_Remove, - VecNested_Slice, -}; - -#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/lib-rt/vecs/vec_t.c b/mypyc/lib-rt/vecs/vec_t.c deleted file mode 100644 index 342143492f26e..0000000000000 --- a/mypyc/lib-rt/vecs/vec_t.c +++ /dev/null @@ -1,502 +0,0 @@ -#ifdef MYPYC_EXPERIMENTAL -// Implementation of generic vec[t], when t is a plain type object (possibly optional). -// -// Examples of types supported: -// -// - vec[str] -// - vec[str | None] -// - vec[object] -// - vec[UserClass] - -#define PY_SSIZE_T_CLEAN -#include -#include "librt_vecs.h" - -static inline VecT vec_error() { - VecT v = { .len = -1 }; - return v; -} - -static inline VecTBufObject *alloc_buf(Py_ssize_t size, size_t item_type) { - VecTBufObject *buf = PyObject_GC_NewVar(VecTBufObject, &VecTBufType, size); - if (buf == NULL) - return NULL; - buf->item_type = item_type; - Py_INCREF(VEC_BUF_ITEM_TYPE(buf)); - PyObject_GC_Track(buf); - return buf; -} - -// Alloc a partially initialized vec. Caller *must* immediately initialize len, and buf->items -// if size > 0. -static VecT vec_alloc(Py_ssize_t size, size_t item_type) { - VecTBufObject *buf; - - if (size == 0) { - buf = NULL; - } else { - buf = alloc_buf(size, item_type); - if (buf == NULL) - return vec_error(); - } - return (VecT) { .buf = buf }; -} - -// Box a VecT value, stealing 'vec'. On failure, return NULL and decref 'vec'. -PyObject *VecT_Box(VecT vec, size_t item_type) { - // An unboxed empty vec may have a NULL buf, but a boxed vec must have it - // allocated, since it contains the item type - if (vec.buf == NULL) { - vec.buf = alloc_buf(0, item_type); - if (vec.buf == NULL) - return NULL; - } - VecTObject *obj = PyObject_GC_New(VecTObject, &VecTType); - if (obj == NULL) { - // vec.buf is always defined, so no need for a NULL heck - Py_DECREF(vec.buf); - return NULL; - } - obj->vec = vec; - PyObject_GC_Track(obj); - return (PyObject *)obj; -} - -VecT VecT_Unbox(PyObject *obj, size_t item_type) { - if (obj->ob_type == &VecTType) { - VecT result = ((VecTObject *)obj)->vec; - if (result.buf->item_type == item_type) { - VEC_INCREF(result); // TODO: Should we borrow instead? - return result; - } - } - // TODO: Better error message, with name of type - PyErr_SetString(PyExc_TypeError, "vec[t] expected"); - return vec_error(); -} - -VecT VecT_ConvertFromNested(VecNestedBufItem item) { - return (VecT) { item.len, (VecTBufObject *)item.buf }; -} - -VecT VecT_New(Py_ssize_t size, Py_ssize_t cap, size_t item_type) { - if (cap < size) - cap = size; - VecT vec = vec_alloc(cap, item_type); - if (VEC_IS_ERROR(vec)) - return vec; - for (Py_ssize_t i = 0; i < cap; i++) { - vec.buf->items[i] = NULL; - } - vec.len = size; - return vec; -} - -static PyObject *vec_repr(PyObject *self) { - VecTObject *v = (VecTObject *)self; - return Vec_GenericRepr(self, v->vec.buf->item_type, 0, 1); -} - -static PyObject *vec_get_item(PyObject *o, Py_ssize_t i) { - VecT v = ((VecTObject *)o)->vec; - if ((size_t)i < (size_t)v.len) { - PyObject *item = v.buf->items[i]; - Py_INCREF(item); - return item; - } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - PyObject *item = v.buf->items[i + v.len]; - Py_INCREF(item); - return item; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } -} - -VecT VecT_Slice(VecT vec, int64_t start, int64_t end) { - if (start < 0) - start += vec.len; - if (end < 0) - end += vec.len; - if (start < 0) - start = 0; - if (start >= vec.len) - start = vec.len; - if (end < start) - end = start; - if (end > vec.len) - end = vec.len; - int64_t slicelength = end - start; - VecT res = vec_alloc(slicelength, vec.buf->item_type); - if (VEC_IS_ERROR(res)) - return res; - res.len = slicelength; - for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec.buf->items[start + i]; - Py_INCREF(item); - res.buf->items[i] = item; - } - return res; -} - -static PyObject *vec_subscript(PyObject *self, PyObject *item) { - VecT vec = ((VecTObject *)self)->vec; - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) - return NULL; - if ((size_t)i < (size_t)vec.len) { - PyObject *result = vec.buf->items[i]; - Py_INCREF(result); - return result; - } else if ((size_t)i + (size_t)vec.len < (size_t)vec.len) { - PyObject *result = vec.buf->items[i + vec.len]; - Py_INCREF(result); - return result; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return NULL; - } - } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step; - if (PySlice_Unpack(item, &start, &stop, &step) < 0) - return NULL; - Py_ssize_t slicelength = PySlice_AdjustIndices(vec.len, &start, &stop, step); - VecT res = vec_alloc(slicelength, vec.buf->item_type); - if (VEC_IS_ERROR(res)) - return NULL; - res.len = slicelength; - Py_ssize_t j = start; - for (Py_ssize_t i = 0; i < slicelength; i++) { - PyObject *item = vec.buf->items[j]; - Py_INCREF(item); - res.buf->items[i] = item; - j += step; - } - PyObject *result = VecT_Box(res, vec.buf->item_type); - if (result == NULL) { - VEC_DECREF(res); - } - return result; - } else { - PyErr_Format(PyExc_TypeError, "vec indices must be integers or slices, not %.100s", - item->ob_type->tp_name); - return NULL; - } -} - -static int vec_ass_item(PyObject *self, Py_ssize_t i, PyObject *o) { - VecT v = ((VecTObject *)self)->vec; - if (!VecT_ItemCheck(v, o, v.buf->item_type)) - return -1; - if ((size_t)i < (size_t)v.len) { - PyObject *old = v.buf->items[i]; - Py_INCREF(o); - v.buf->items[i] = o; - Py_XDECREF(old); - return 0; - } else if ((size_t)i + (size_t)v.len < (size_t)v.len) { - PyObject *old = v.buf->items[i + v.len]; - Py_INCREF(o); - v.buf->items[i + v.len] = o; - Py_XDECREF(old); - return 0; - } else { - PyErr_SetString(PyExc_IndexError, "index out of range"); - return -1; - } -} - -static PyObject *vec_richcompare(PyObject *self, PyObject *other, int op) { - PyObject *res; - if (op == Py_EQ || op == Py_NE) { - if (other->ob_type != &VecTType) { - res = op == Py_EQ ? Py_False : Py_True; - } else { - VecT x = ((VecTObject *)self)->vec; - VecT y = ((VecTObject *)other)->vec; - if (x.buf->item_type != y.buf->item_type) { - res = op == Py_EQ ? Py_False : Py_True; - } else { - // TODO: why pointers to len? - return Vec_GenericRichcompare(&x.len, x.buf->items, &y.len, y.buf->items, op); - } - } - } else - res = Py_NotImplemented; - Py_INCREF(res); - return res; -} - -// Append item to 'vec', stealing 'vec'. Return 'vec' with item appended. -VecT VecT_Append(VecT vec, PyObject *x, size_t item_type) { - if (vec.buf == NULL) { - VecT new = vec_alloc(1, item_type); - if (VEC_IS_ERROR(new)) - return new; - Py_INCREF(x); - new.len = 1; - new.buf->items[0] = x; - return new; - } - Py_ssize_t cap = VEC_CAP(vec); - Py_INCREF(x); - if (vec.len < cap) { - vec.buf->items[vec.len] = x; - vec.len++; - return vec; - } else { - Py_ssize_t new_size = 2 * cap + 1; - // TODO: Avoid initializing to zero here - VecT new = vec_alloc(new_size, vec.buf->item_type); - if (VEC_IS_ERROR(new)) { - Py_DECREF(x); - // The input vec is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(vec); - return new; - } - // Copy items to new vec. - memcpy(new.buf->items, vec.buf->items, sizeof(PyObject *) * vec.len); - memset(new.buf->items + vec.len, 0, sizeof(PyObject *) * (new_size - vec.len)); - // Clear the items in the old vec. We avoid reference count manipulation. - memset(vec.buf->items, 0, sizeof(PyObject *) * vec.len); - new.buf->items[vec.len] = x; - new.len = vec.len + 1; - VEC_DECREF(vec); - return new; - } -} - -// Remove item from 'vec', stealing 'vec'. Return 'vec' with item removed. -VecT VecT_Remove(VecT v, PyObject *arg) { - PyObject **items = v.buf->items; - for (Py_ssize_t i = 0; i < v.len; i++) { - int match = 0; - if (items[i] == arg) - match = 1; - else { - int itemcmp = PyObject_RichCompareBool(items[i], arg, Py_EQ); - if (itemcmp < 0) { - // The input v is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(v); - return vec_error(); - } - match = itemcmp; - } - if (match) { - if (i < v.len - 1) { - Py_CLEAR(items[i]); - for (; i < v.len - 1; i++) { - items[i] = items[i + 1]; - } - // Keep a duplicate item, since there could be another reference - // to the buffer with a longer length, and they expect a valid reference. - Py_XINCREF(items[v.len - 1]); - } - v.len--; - // Return the stolen reference without INCREF - return v; - } - } - PyErr_SetString(PyExc_ValueError, "vec.remove(x): x not in vec"); - // The input v is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(v); - return vec_error(); -} - -// Pop item from 'vec', stealing 'vec'. Return struct with modified 'vec' and the popped item. -VecTPopResult VecT_Pop(VecT v, Py_ssize_t index) { - VecTPopResult result; - - if (index < 0) - index += v.len; - - if (index < 0 || index >= v.len) { - PyErr_SetString(PyExc_IndexError, "index out of range"); - // The input v is being consumed/stolen by this function, so on error - // we must decref it to avoid leaking the buffer. - VEC_DECREF(v); - result.f0 = vec_error(); - result.f1 = NULL; - return result; - } - - PyObject **items = v.buf->items; - result.f1 = items[index]; - for (Py_ssize_t i = index; i < v.len - 1; i++) - items[i] = items[i + 1]; - // Keep duplicate item, since there could be another reference - // to the buffer with a longer length, and they expect a valid reference. - Py_XINCREF(items[v.len - 1]); - v.len--; - // Return the stolen reference without INCREF - result.f0 = v; - return result; -} - -static int -VecT_traverse(VecTObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->vec.buf); - return 0; -} - -static int -VecT_clear(VecTObject *self) -{ - Py_CLEAR(self->vec.buf); - return 0; -} - -static void -VecT_dealloc(VecTObject *self) -{ - PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecT_dealloc) - Py_CLEAR(self->vec.buf); - Py_TYPE(self)->tp_free((PyObject *)self); - Py_TRASHCAN_END -} - -static int -VecTBuf_traverse(VecTBufObject *self, visitproc visit, void *arg) -{ - Py_VISIT(VEC_BUF_ITEM_TYPE(self)); - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_VISIT(self->items[i]); - } - return 0; -} - -static int -VecTBuf_clear(VecTBufObject *self) -{ - if (self->item_type) { - Py_DECREF(VEC_BUF_ITEM_TYPE(self)); - self->item_type = 0; - } - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_CLEAR(self->items[i]); - } - return 0; -} - -static void -VecTBuf_dealloc(VecTBufObject *self) -{ - PyObject_GC_UnTrack(self); - Py_TRASHCAN_BEGIN(self, VecTBuf_dealloc) - if (self->item_type) { - Py_DECREF(VEC_BUF_ITEM_TYPE(self)); - self->item_type = 0; - } - for (Py_ssize_t i = 0; i < VEC_BUF_SIZE(self); i++) { - Py_CLEAR(self->items[i]); - } - Py_TYPE(self)->tp_free((PyObject *)self); - Py_TRASHCAN_END -} - -static Py_ssize_t vec_length(PyObject *o) { - return ((VecTObject *)o)->vec.len; -} - -static PyMappingMethods VecTMapping = { - .mp_length = vec_length, - .mp_subscript = vec_subscript, -}; - -static PySequenceMethods VecTSequence = { - .sq_item = vec_get_item, - .sq_ass_item = vec_ass_item, -}; - -static PyMethodDef vec_methods[] = { - {NULL, NULL, 0, NULL}, /* Sentinel */ -}; - -PyTypeObject VecTBufType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vecbuf", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecTBufObject) - sizeof(PyObject *), - .tp_itemsize = sizeof(PyObject *), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecTBuf_traverse, - //.tp_new = vecbuf_i64_new, //?? - .tp_free = PyObject_GC_Del, - .tp_clear = (inquiry)VecTBuf_clear, - .tp_dealloc = (destructor)VecTBuf_dealloc, -}; - -PyTypeObject VecTType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "vec", - .tp_doc = "vec doc", - .tp_basicsize = sizeof(VecTObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, - .tp_traverse = (traverseproc)VecT_traverse, - .tp_clear = (inquiry)VecT_clear, - .tp_dealloc = (destructor)VecT_dealloc, - //.tp_free = PyObject_GC_Del, - .tp_repr = (reprfunc)vec_repr, - .tp_as_sequence = &VecTSequence, - .tp_as_mapping = &VecTMapping, - .tp_richcompare = vec_richcompare, - .tp_methods = vec_methods, - // TODO: free -}; - -PyObject *VecT_FromIterable(size_t item_type, PyObject *iterable) { - VecT v = vec_alloc(0, item_type); - if (VEC_IS_ERROR(v)) - return NULL; - v.len = 0; - - PyObject *iter = PyObject_GetIter(iterable); - if (iter == NULL) { - VEC_DECREF(v); - return NULL; - } - PyObject *item; - while ((item = PyIter_Next(iter)) != NULL) { - if (!VecT_ItemCheck(v, item, item_type)) { - Py_DECREF(iter); - VEC_DECREF(v); - Py_DECREF(item); - return NULL; - } - v = VecT_Append(v, item, item_type); - Py_DECREF(item); - if (VEC_IS_ERROR(v)) { - Py_DECREF(iter); - VEC_DECREF(v); - return NULL; - } - } - Py_DECREF(iter); - if (PyErr_Occurred()) { - VEC_DECREF(v); - return NULL; - } - return VecT_Box(v, item_type); -} - -VecTAPI Vec_TAPI = { - &VecTType, - &VecTBufType, - VecT_New, - VecT_Box, - VecT_Unbox, - VecT_ConvertFromNested, - VecT_Append, - VecT_Pop, - VecT_Remove, - VecT_Slice, -}; - -#endif // MYPYC_EXPERIMENTAL diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index 1e721f94c783f..a9627b7dad1d3 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -185,8 +185,6 @@ def test_equality_different_types() -> None: assert_neq(vec[i64]([1]), None) assert_neq(vec[i64]([1]), [1]) assert_neq(vec[i64]([1]), (1,)) - assert_neq(vec[i64]([1]), vec[object]([1])) - assert_neq(vec[i64](), vec[object]()) def assert_eq(x: object, y: object) -> None: assert x == y diff --git a/mypyc/test-data/run-vecs-nested-interp.test b/mypyc/test-data/run-vecs-nested-interp.test deleted file mode 100644 index b78ecff51b415..0000000000000 --- a/mypyc/test-data/run-vecs-nested-interp.test +++ /dev/null @@ -1,317 +0,0 @@ -[case testLibrtVecsNestedInterpreted_librt_experimental] -# Test cases for nested vecs, using generic operations (simulates use from interpreted code). - -import sys -from typing import Any -import typing - -import librt.vecs -import mypy_extensions - -from testutil import assertRaises - -vec: Any = librt.vecs.vec -append: Any = librt.vecs.append -remove: Any = librt.vecs.remove -pop: Any = librt.vecs.pop -getsizeof: Any = getattr(sys, "getsizeof") -i64: Any = mypy_extensions.i64 - -# Work around test stub limitations -Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") - - -def test_vec_indexing() -> None: - assert type(vec[vec[str]]) is type(vec[str]) - assert type(vec[vec[Optional[str]]]) is type(vec[str]) - assert type(vec[vec[i64]]) is type(vec[str]) - assert type(vec[vec[vec[vec[vec[vec[str]]]]]]) is type(vec[str]) - with assertRaises(TypeError): - vec[Optional[vec[str]]] - with assertRaises(TypeError): - vec[Optional[vec[Optional[str]]]] - -def test_type_str() -> None: - assert str(vec[Optional[str]]) == "" - assert str(vec[vec[str]]) == "" - assert str(vec[vec[i64]]) == "" - assert str(vec[vec[vec[vec[vec[str]]]]]) == "" - assert str( - vec[vec[vec[Optional[str]]]]) == "" - -def test_construct_empty_nested() -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v2 = vec[vec[vec[vec[vec[str]]]]]() - assert len(v2) == 0 - v3 = vec[vec[i64]]() - assert len(v3) == 0 - -def test_construct_empty_nested_optional() -> None: - v = vec[vec[Optional[str]]]() - assert len(v) == 0 - -def test_construct_from_initializer_nested() -> None: - v = vec[vec[str]]([vec[str](['xyz'])]) - assert len(v) == 1 - assert v[0][0] == 'xyz' - with assertRaises(TypeError): - vec[str](1) - with assertRaises(TypeError): - vec[str]([1]) - with assertRaises(TypeError): - vec[str]([vec[bytes]()]) - -def test_repr() -> None: - assert str(vec[vec[str]]()) == "vec[vec[str]]([])" - assert str(vec[vec[Optional[str]]]()) == "vec[vec[str | None]]([])" - -def test_append_nested() -> None: - v = vec[vec[str]]() - vv = append(vec[str](), '1') - v = append(v, vv) - assert str(v) == "vec[vec[str]]([['1']])" - for n in range(2, 10): - vv = append(vec[str](), str(n)) - v = append(v, vv) - assert str(v) == ( - "vec[vec[str]]([['1'], ['2'], ['3'], ['4'], ['5'], ['6'], ['7'], ['8'], ['9']])" - ) - -def test_append_i64() -> None: - v = vec[vec[i64]]() - vv = append(vec[i64](), 1) - v = append(v, vv) - assert str(v) == "vec[vec[i64]]([[1]])" - for n in range(2, 10): - vv = append(vec[i64](), n) - v = append(v, vv) - assert str(v) == ( - "vec[vec[i64]]([[1], [2], [3], [4], [5], [6], [7], [8], [9]])" - ) - -def test_append_nested_optional() -> None: - v = vec[vec[Optional[str]]]() - v = append(v, vec[Optional[str]]()) - v = append(v, vec[Optional[str]](['x', None])) - assert str(v) == "vec[vec[str | None]]([[], ['x', None]])" - -def test_size() -> None: - v = vec[vec[str]]() - assert getsizeof(v) == 40 - v = append(v, vec[str]()) - assert getsizeof(v) == 48 # TODO: Why different size? - -def test_get_item() -> None: - v = append(vec[vec[str]](), append(vec[str](), 'x')) - assert repr(v[0]) == "vec[str](['x'])" - v = append(v, append(vec[str](), 'y')) - assert repr(v[0]) == "vec[str](['x'])" - assert repr(v[1]) == "vec[str](['y'])" - vv = append(vec[vec[Optional[str]]](), vec[Optional[str]]()) - assert vv[0] == vec[Optional[str]]() - -def test_get_item_error() -> None: - v = append(vec[vec[str]](), vec[str](['1'])) - v = append(v, vec[str](['2'])) - v = append(v, vec[str](['3'])) - with assertRaises(IndexError): - v[3] - with assertRaises(IndexError): - v[4] - n = 2 ** 100 - with assertRaises(IndexError): - v[n] - -def test_get_item_negative() -> None: - v = vec[vec[str]]([vec[str](['2']), vec[str](['5']), vec[str](['7'])]) - assert v[-1] == vec[str](['7']) - assert v[-2] == vec[str](['5']) - assert v[-3] == vec[str](['2']) - with assertRaises(IndexError): - v[-4] - with assertRaises(IndexError): - v[-2**62] - -def test_len() -> None: - v = vec[vec[str]]() - assert len(v) == 0 - v = append(v, vec[str](['1'])) - assert len(v) == 1 - v = append(v, vec[str]()) - assert len(v) == 2 - for i in range(100): - v = append(v, vec[str](['i'])) - assert len(v) == i + 3 - -def test_set_item() -> None: - v = vec[vec[str]]() - v = append(v, vec[str](['1'])) - v = append(v, vec[str](['2'])) - v[0] = vec[str](['3']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['2']) - v[1] = vec[str](['5']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['5']) - -def test_set_item_error_nested() -> None: - v = append(vec[vec[str]](), vec[str]()) - v = append(v, vec[str]()) - v = append(v, vec[str]()) - with assertRaises(IndexError): - v[3] = vec[str]() - with assertRaises(TypeError): - v[0] = 'x' - with assertRaises(TypeError): - v[0] = vec[vec[str]]() - with assertRaises(TypeError): - v[0] = vec[Optional[str]]() - assert repr(v[0]) == 'vec[str]([])' - -def test_set_item_negative() -> None: - v = vec[vec[str]]() - v = append(v, vec[str](['1'])) - v = append(v, vec[str](['2'])) - v[-2] = vec[str](['3']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['2']) - v[-1] = vec[str](['5']) - assert v[0] == vec[str](['3']) - assert v[1] == vec[str](['5']) - with assertRaises(IndexError): - v[-3] = vec[str](['5']) - -def test_equality() -> None: - assert_eq(vec[vec[str]](), vec[vec[str]]()) - assert_eq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['x'])])) - assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]()) - assert_neq(vec[vec[str]]([vec[str](['x'])]), vec[vec[str]]([vec[str](['y'])])) - -def test_equality_different_types() -> None: - assert_neq(vec[vec[str]](), vec[str]()) - assert_neq(vec[vec[str]](), vec[vec[vec[str]]]()) - assert_neq(vec[vec[Optional[str]]](), vec[vec[str]]()) - -def assert_eq(x: object, y: object) -> None: - assert x == y - assert y == x - b = x != y - assert not b - b = y != x - assert not b - -def assert_neq(x: object, y: object) -> None: - assert x != y - assert y != x - b = x == y - assert not b - b = y == x - assert not b - -def test_slicing() -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[1:4] == vec[vec[str]]([v[1], v[2], v[3]]) - assert v[2:-1] == vec[vec[str]]([v[2], v[3]]) - assert v[-2:-1] == vec[vec[str]]([v[3]]) - assert v[1:] == vec[vec[str]]([v[1], v[2], v[3], v[4]]) - assert v[:-1] == vec[vec[str]]([v[0], v[1], v[2], v[3]]) - assert v[:] == v - assert v[:] is not v - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[vec[str]]([v[2], v[3], v[4]]) - assert v[-100:2] == vec[vec[str]]([v[0], v[1]]) - assert v[5:100] == vec[vec[str]]([]) - assert v[50:100] == vec[vec[str]]([]) - assert v[-100:-50] == vec[vec[str]]([]) - -def test_slicing_with_step() -> None: - v = vec[vec[str]](vec[str]([str(i)]) for i in range(10)) - assert v[1:5:2] == vec[vec[str]]([v[1], v[3]]) - assert v[1:6:2] == vec[vec[str]]([v[1], v[3], v[5]]) - assert v[5:1:-2] == vec[vec[str]]([v[5], v[3]]) - assert v[6:1:-2] == vec[vec[str]]([v[6], v[4], v[2]]) - v = vec[vec[str]](vec[str]([str(i)]) for i in range(5)) - assert v[::-1] == vec[vec[str]]([v[4], v[3], v[2], v[1], v[0]]) - with assertRaises(ValueError): - v[1:3:0] - -def test_slicing_with_different_item_types() -> None: - v = vec[Optional[str]](['x', None, 'y']) - assert v[1:] == vec[Optional[str]]([None, 'y']) - vv = vec[vec[str]]([vec[str](['x']), vec[str](['y'])]) - assert vv[1:] == vec[vec[str]]([vec[str](['y'])]) - -def test_remove() -> None: - a = ['4x', '7x', '9x'] - vv = [vec[str]([s]) for s in a] - for i, s in enumerate(a): - v = vec[vec[str]](vv) - v = remove(v, vv[i]) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) - v = vec[vec[str]](vv) - # Make sure we have a different object identity - v = remove(v, vec[str]([a[i]])) - assert v == vec[vec[str]]([j for j in vv if j != vv[i]]) - v = vec[vec[str]](vv) - v = remove(v, vv[0]) - v = remove(v, vv[1]) - v = remove(v, vv[2]) - assert v == vec[vec[str]]() - with assertRaises(ValueError): - remove(v, vec[str](['4'])) - v = append(v, vec[str](['5'])) - with assertRaises(ValueError): - remove(v, vec[str](['7'])) - v = remove(v, vec[str](['5'])) - assert len(v) == 0 - v = v0 = vec[vec[str]]([vec[str](['x']) for _ in range(3)]) - v = remove(v, vec[str](['x'])) - assert v == v0[1:] - v = remove(v, vec[str](['x'])) - assert v == v0[2:] - v = remove(v, vec[str](['x'])) - assert v == vec[vec[str]]() - vb: Any = vec[bytes]([b'x']) - with assertRaises(TypeError): - remove(v, vb) - -def test_pop_last() -> None: - v = vec[vec[str]]([vec[str](['4']), vec[str](['5']), vec[str](['9'])]) - v, item = pop(v) - assert item == vec[str](['9']) - assert v == vec[vec[str]]([vec[str](['4']), vec[str](['5'])]) - v, item = pop(v) - assert item == vec[str](['5']) - assert v == vec[vec[str]]([vec[str](['4'])]) - v, item = pop(v) - assert item == vec[str](['4']) - assert v == vec[vec[str]]() - with assertRaises(IndexError): - pop(v) - -def test_pop_index() -> None: - v = vec[vec[str]](vec[str]([s]) for s in ['4', '7', '9', '15', '22']) - v, item = pop(v, 0) - assert item == vec[str](['4']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15', '22']) - v, item = pop(v, -1) - assert item == vec[str](['22']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '9', '15']) - v, item = pop(v, 1) - assert item == vec[str](['9']) - assert v == vec[vec[str]](vec[str]([s]) for s in ['7', '15']) - - with assertRaises(IndexError): - pop(v, 2) - - with assertRaises(IndexError): - pop(v, -3) - - v, item = pop(v, -2) - assert item == vec[str](['7']) - assert v == vec[vec[str]]([vec[str](['15'])]) - v, item = pop(v, 0) - assert item == vec[str](['15']) - assert v == vec[vec[str]]() diff --git a/mypyc/test-data/run-vecs-t-interp.test b/mypyc/test-data/run-vecs-t-interp.test deleted file mode 100644 index eae5898be861c..0000000000000 --- a/mypyc/test-data/run-vecs-t-interp.test +++ /dev/null @@ -1,457 +0,0 @@ -[case testLibrtVecsTInterpreted_librt_experimental] -# Test cases for vec[], using generic operations (simulates use from interpreted code). -import sys -from typing import cast, List, Any -import typing - -import librt.vecs -import mypy_extensions - -from testutil import assertRaises -from helpers import ClassWithCustomMeta - -vec: Any = librt.vecs.vec -append: Any = librt.vecs.append -remove: Any = librt.vecs.remove -pop: Any = librt.vecs.pop -getsizeof: Any = getattr(sys, "getsizeof") -i64: Any = mypy_extensions.i64 -version_info: Any = sys.version_info - -# Work around test stub limitations -Optional: Any = getattr(getattr(sys, "modules")["typing"], "Optional") - -def assert_eq(x: object, y: object) -> None: - assert x == y - assert y == x - b = x != y - assert not b - b = y != x - assert not b - -def assert_neq(x: object, y: object) -> None: - assert x != y - assert y != x - b = x == y - assert not b - b = y == x - assert not b - -@mypy_extensions.mypyc_attr(native_class=False) -class StringSubclass(str): - pass - -class ClassWithRaisingEq: - def __eq__(self, other): - raise RuntimeError() - -def test_vec_indexing() -> None: - assert vec[str] is not vec[i64] - assert type(vec[str]) is not type - assert type(vec[Optional[str]]) is type(vec[str]) - -def test_type_str() -> None: - assert str(vec[str]) == "" - assert str(vec[Optional[str]]) == "" - -def test_construct_empty() -> None: - v = vec[str]() - assert len(v) == 0 - v2 = vec[Optional[str]]() - assert len(v2) == 0 - assert type(v) is type(v2) - -def test_construct_or_union() -> None: - if version_info >= (3, 10): - v = vec[str | None]() - assert len(v) == 0 - assert str(v) == 'vec[str | None]([])' - -def test_construct_from_initializer() -> None: - v = vec[str](['xyz']) - assert len(v) == 1 - assert v[0] == 'xyz' - v = vec[str](x + '!' for x in ['x', '', 'foo', 'bar']) - assert len(v) == 4 - assert v[0] == 'x!' - assert v[1] == '!' - assert v[2] == 'foo!' - assert v[3] == 'bar!' - with assertRaises(TypeError): - vec[str](1) - with assertRaises(TypeError): - vec[str]([1]) - with assertRaises(TypeError): - vec[str]([None]) - -def test_construct_from_initializer_optional() -> None: - v = vec[Optional[str]]([None, 'xyz']) - assert len(v) == 2 - assert v[0] is None - assert v[1] == 'xyz' - v = vec[Optional[str]](x for x in ['x', None, 'foo', 'bar']) - assert len(v) == 4 - assert v[0] == 'x' - assert v[1] is None - assert v[2] == 'foo' - assert v[3] == 'bar' - with assertRaises(TypeError): - vec[Optional[str]](1) - with assertRaises(TypeError): - vec[Optional[str]]([1]) - -def test_append() -> None: - v = vec[str]() - v = append(v, '1') - assert str(v) == "vec[str](['1'])" - for n in range(2, 10): - v = append(v, str(n)) - assert str(v) == "vec[str](['1', '2', '3', '4', '5', '6', '7', '8', '9'])" - - v2 = vec[Optional[str]]() - v2 = append(v2, 'x') - v2 = append(v2, None) - assert str(v2) == "vec[str | None](['x', None])" - -def test_append_error() -> None: - v: Any = vec[str]() - with assertRaises(TypeError): - v = append(v, b'x') - with assertRaises(TypeError): - v = append(v, ['x']) - with assertRaises(TypeError): - v = append(v, None) - assert len(v) == 0 - - v2: Any = vec[Optional[str]]() - with assertRaises(TypeError): - v2 = append(v2, b'x') - -def test_repr() -> None: - assert str(vec[str]()) == "vec[str]([])" - assert str(vec[Optional[str]]([None, 'x'])) == "vec[str | None]([None, 'x'])" - -def test_size() -> None: - v = vec[str]() - assert getsizeof(v) == 48 - v = append(v, '123') - assert getsizeof(v) == 48 - -def test_get_item() -> None: - v = append(vec[str](), '44') - assert v[0] == '44' - v = append(v, '56') - assert v[0] == '44' - assert v[1] == '56' - - v2 = vec[Optional[str]]([None, '12']) - assert v2[0] is None - assert v2[1] == '12' - -def test_get_item_error() -> None: - v = append(vec[str](), '1') - v = append(v, '2') - v = append(v, '3') - with assertRaises(IndexError): - v[3] - with assertRaises(IndexError): - v[-4] - n = 2 ** 100 - with assertRaises(IndexError): - v[n] - -def test_get_item_negative() -> None: - v = vec[str](['2', '5', '7']) - assert v[-1] == '7' - assert v[-2] == '5' - assert v[-3] == '2' - with assertRaises(IndexError): - v[-4] - with assertRaises(IndexError): - v[-2**62] - -def test_len() -> None: - v = vec[str]() - assert len(v) == 0 - v = append(v, '1') - assert len(v) == 1 - v = append(v, '2') - assert len(v) == 2 - for i in range(100): - v = append(v, 'i') - assert len(v) == i + 3 - -def test_set_item() -> None: - v = vec[str]() - v = append(v, '1') - v = append(v, '2') - v[0] = '3' - assert v[0] == '3' - assert v[1] == '2' - v[1] = '5' - assert v[0] == '3' - assert v[1] == '5' - -def test_set_item_optional() -> None: - v = vec[Optional[str]]() - v = append(v, '1') - v = append(v, None) - v[0] = '3' - assert v[0] == '3' - assert v[1 + int()] is None # TODO: The '+ int()' is due to mypy type narrowing bug - v[1] = '5' - assert v[0] == '3' - assert v[1] == '5' - v[1] = None - assert v[0] == '3' - assert v[1] is None - -def test_set_item_error() -> None: - v = append(vec[str](), '1') - v = append(v, '2') - v = append(v, '3') - with assertRaises(IndexError): - v[3] = '1' - with assertRaises(TypeError): - v[0] = b'x' - with assertRaises(TypeError): - v[0] = vec[str]() - with assertRaises(TypeError): - v[0] = None - assert v[0] == '1' - -def test_set_item_error_optional() -> None: - v = append(vec[Optional[str]](), '1') - v = append(v, None) - v = append(v, '3') - with assertRaises(IndexError): - v[3] = '1' - with assertRaises(IndexError): - v[3] = None - with assertRaises(IndexError): - v[-4] = '1' - with assertRaises(IndexError): - v[-4] = None - with assertRaises(TypeError): - v[0] = b'x' - with assertRaises(TypeError): - v[0] = vec[str]() - with assertRaises(TypeError): - v[0] = vec[Optional[str]]() - assert v[0] == '1' - -def test_set_item_negative() -> None: - v = vec[str]() - v = append(v, '1') - v = append(v, '2') - v[-2] = '3' - assert v[0] == '3' - assert v[1] == '2' - v[-1] = '5' - assert v[0] == '3' - assert v[1] == '5' - with assertRaises(IndexError): - v[-3] = '1' - -def test_nested_in_type() -> None: - cast(List[vec[str]], []) - -def test_cannot_overload_isinstance() -> None: - assert isinstance('', ClassWithCustomMeta) - v = vec[ClassWithCustomMeta]() - v = append(v, ClassWithCustomMeta()) - with assertRaises(TypeError): - v = append(v, '') - with assertRaises(TypeError): - v[0] = '' - -def test_add_subclass_instance() -> None: - v = vec[str]() - v = append(v, StringSubclass()) - v[0] = StringSubclass() - - v2 = vec[StringSubclass]() - with assertRaises(TypeError): - v2 = append(v2, '') - -def test_equality() -> None: - assert_eq(vec[str](), vec[str]()) - assert_eq(vec[str](['x']), vec[str](['x'])) - v = vec[str](['x']) - assert_eq(v, v) - assert_eq(vec[str](['x', 'y']), vec[str](['x', 'y'])) - - assert_neq(vec[str](), vec[str](['x'])) - assert_neq(vec[str](['x', 'y']), vec[str](['x'])) - assert_neq(vec[str](['x']), vec[str](['y'])) - assert_neq(vec[str](['x', 'y']), vec[str](['y', 'y'])) - assert_neq(vec[str](['x', 'y']), vec[str](['x', 'x'])) - - assert_eq(vec[Optional[str]](), vec[Optional[str]]()) - assert_eq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', None])) - assert_neq(vec[Optional[str]](['x', None]), vec[Optional[str]](['x', 'y'])) - -def test_equality_different_types() -> None: - assert_neq(vec[str](['x']), ['x']) - assert_neq(vec[str](['x']), ('x',)) - assert_neq(vec[str](['x']), vec[object](['x'])) - assert_neq(vec[str](), vec[object]()) - assert_neq(vec[str](), vec[Optional[str]]()) - -def test_exception_from_eq() -> None: - with assertRaises(RuntimeError): - vec[ClassWithRaisingEq]([ClassWithRaisingEq()]) == vec[ClassWithRaisingEq]([ClassWithRaisingEq()]) - -def test_iteration() -> None: - for x in vec[str](): - assert False - a = [] - for x in vec[str](['x']): - a.append(x) - assert a == ['x'] - a = [] - for x in vec[str](['11', '23', '45', '64']): - a.append(x) - assert a == ['11', '23', '45', '64'] - a = [] - for x in vec[Optional[str]](['11', None, '45', '64']): - a.append(x) - assert a == ['11', None, '45', '64'] - -def test_slicing() -> None: - v = vec[str](str(i) for i in range(5)) - assert v[1:4] == vec[str](['1', '2', '3']) - assert v[2:-1] == vec[str](['2', '3']) - assert v[-2:-1] == vec[str](['3']) - assert v[1:] == vec[str](['1', '2', '3', '4']) - assert v[:-1] == vec[str](['0', '1', '2', '3']) - assert v[:] == v - assert v[:] is not v - assert v[0:5] ==v - assert v[0:5] is not v - assert v[2:100] == vec[str](['2', '3', '4']) - assert v[-100:2] == vec[str](['0', '1']) - assert v[5:100] == vec[str]([]) - assert v[50:100] == vec[str]([]) - assert v[-100:-50] == vec[str]([]) - -def test_slicing_optional() -> None: - v = vec[Optional[str]](['x', None, 'y']) - assert v[:2] == vec[Optional[str]](['x', None]) - -def test_slicing_with_step() -> None: - v = vec[str](str(i) for i in range(10)) - assert v[1:5:2] == vec[str](['1', '3']) - assert v[1:6:2] == vec[str](['1', '3', '5']) - assert v[5:1:-2] == vec[str](['5', '3']) - assert v[6:1:-2] == vec[str](['6', '4', '2']) - v = vec[str](str(i) for i in range(5)) - assert v[::-1] == vec[str](['4', '3', '2', '1', '0']) - with assertRaises(ValueError): - v[1:3:0] - -def test_contains() -> None: - v = vec[str](str(i) for i in range(2, 7)) - for i in range(2, 7): - assert str(i) in v - assert 'x' not in v - assert '22' not in v - vv = vec[str]() - assert '3' not in vv - - v2 = vec[Optional[str]]() - assert '3' not in v2 - assert None not in v2 - v2 = vec[Optional[str]](['3', None]) - assert '3' in v2 - assert None in v2 - assert '4' not in v2 - -def test_remove() -> None: - a = ['4x', '7x', '9x'] - for i in a: - v = vec[str](a) - v = remove(v, i) - assert v == vec[str]([j for j in a if j != i]) - v = vec[str](a) - # Make sure we have a different id - v = remove(v, (i + 'x')[:-1]) - assert v == vec[str]([j for j in a if j != i]) - v = vec[str](a) - v = remove(v, '4x') - v = remove(v, '7x') - v = remove(v, '9x') - assert v == vec[str]() - with assertRaises(ValueError): - remove(v, '4') - v = append(v, '5') - with assertRaises(ValueError): - remove(v, '7') - v = remove(v, '5') - assert len(v) == 0 - v = vec[str](['x', 'x', 'x']) - v = remove(v, 'x') - assert v == vec[str](['x', 'x']) - v = remove(v, 'x') - assert v == vec[str](['x']) - v = remove(v, 'x') - assert v == vec[str]() - b: Any = b'x' - with assertRaises(TypeError): - remove(v, b) - -def test_remove_optional() -> None: - v = vec[Optional[str]](['x', None]) - v = remove(v, 'x') - assert str(v) == "vec[str | None]([None])" - v = remove(v, None) - assert str(v) == "vec[str | None]([])" - -def test_pop_last() -> None: - v = vec[str](['4', '7', '9']) - v, item = pop(v) - assert item == '9' - assert v == vec[str](['4', '7']) - v, item = pop(v) - assert item == '7' - assert v == vec[str](['4']) - v, item = pop(v) - assert item == '4' - assert v == vec[str]() - with assertRaises(IndexError): - pop(v) - -def test_pop_index() -> None: - v = vec[str](['4', '7', '9', '15', '22']) - v, item = pop(v, 0) - assert item == '4' - assert v == vec[str](['7', '9', '15', '22']) - v, item = pop(v, -1) - assert item == '22' - assert v == vec[str](['7', '9', '15']) - v, item = pop(v, 1) - assert item == '9' - assert v == vec[str](['7', '15']) - - with assertRaises(IndexError): - pop(v, 2) - - with assertRaises(IndexError): - pop(v, -3) - - v, item = pop(v, -2) - assert item == '7' - assert v == vec[str](['15']) - v, item = pop(v, 0) - assert item == '15' - assert v == vec[str]() - -[file helpers.py] -# Test helper classes - -class MetaWithInstanceCheck(type): - def __instancecheck__(cls, instance): - return isinstance(instance, str) - -class ClassWithCustomMeta(metaclass=MetaWithInstanceCheck): - pass diff --git a/mypyc/test/test_run.py b/mypyc/test/test_run.py index c6aa034b3f7e8..9dbd3db1e8a8e 100644 --- a/mypyc/test/test_run.py +++ b/mypyc/test/test_run.py @@ -80,8 +80,6 @@ "run-match.test", "run-vecs-i64-interp.test", "run-vecs-misc-interp.test", - "run-vecs-t-interp.test", - "run-vecs-nested-interp.test", ] if sys.version_info >= (3, 12): From 9cbf307fa57b4a093dec872d7028fb222270a4f8 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 12:57:45 +0000 Subject: [PATCH 201/208] Fix typos in comments --- mypyc/lib-rt/vecs/librt_vecs.c | 2 +- mypyc/lib-rt/vecs/librt_vecs.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 77c9837a1352f..276c71c0133de 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -84,7 +84,7 @@ PyTypeObject *LibRTVecs_U8TypeObj; // vec generic alias // -// Used for the result of vec[t] (indexing) in interpeted context that must preserve +// Used for the result of vec[t] (indexing) in interpreted context that must preserve // knowledge of 't'. These aren't real types. This only supports constructing instances. typedef struct { PyObject_HEAD diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index 7a9ecb84b2c85..a2336c0ac0772 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -415,7 +415,7 @@ extern PyTypeObject VecU8Type; extern PyTypeObject VecFloatType; extern PyTypeObject VecBoolType; -// Type objects correponding the 'i64', 'i32', 'i16, and 'u8' types +// Type objects corresponding to the 'i64', 'i32', 'i16, and 'u8' types extern PyTypeObject *LibRTVecs_I64TypeObj; extern PyTypeObject *LibRTVecs_I32TypeObj; extern PyTypeObject *LibRTVecs_I16TypeObj; From a3761af852f7d1e415ef3c6c77afee9d662609d0 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 13:04:30 +0000 Subject: [PATCH 202/208] Fix test on free threading build --- mypyc/test-data/fixtures/testutil.py | 5 +++++ mypyc/test-data/run-vecs-i64-interp.test | 9 +++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index b2700f731ddd6..a0e0c0c04534e 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -3,6 +3,7 @@ from contextlib import contextmanager from collections.abc import Iterator import math +import sys from typing import ( Any, Iterator, TypeVar, Generator, Optional, List, Tuple, Sequence, Union, Callable, Awaitable, Generic @@ -101,3 +102,7 @@ def make_python_function(f: F) -> F: def g(*args: Any, **kwargs: Any) -> Any: return f(*args, **kwargs) return g # type: ignore + + +def is_gil_disabled() -> bool: + return hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index a9627b7dad1d3..86b60f7fd0335 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -13,7 +13,7 @@ import typing import librt.vecs import mypy_extensions -from testutil import assertRaises +from testutil import assertRaises, is_gil_disabled # Access via Any references to force generic operations vec: Any = librt.vecs.vec @@ -81,9 +81,10 @@ def test_repr() -> None: def test_size() -> None: v = vec[i64]() - assert getsizeof(v) == 32 - v = append(v, 123) - assert getsizeof(v) == 32 + if not is_gil_disabled(): + assert getsizeof(v) == 32 + v = append(v, 123) + assert getsizeof(v) == 32 def test_get_item() -> None: v = append(vec[i64](), 44) From ef31b679c6945f74d7195693f74ac0b17c8a473e Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 13:08:54 +0000 Subject: [PATCH 203/208] Fix compile error on 32-bit windows --- mypyc/lib-rt/vecs/librt_vecs.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.h b/mypyc/lib-rt/vecs/librt_vecs.h index a2336c0ac0772..f29d696bac665 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.h +++ b/mypyc/lib-rt/vecs/librt_vecs.h @@ -17,6 +17,7 @@ import_librt_vecs(void) #define PY_SSIZE_T_CLEAN #include +#include // Magic (native) integer return value on exception. Caller must also // use PyErr_Occurred() since this overlaps with valid integer values. @@ -475,7 +476,7 @@ static inline int32_t VecI32_UnboxItem(PyObject *o) { if (Vec_CheckFloatError(o)) return -1; long x = PyLong_AsLong(o); - if (x >= 0x80000000L || x < -0x80000000L) { + if (x > INT32_MAX || x < INT32_MIN) { PyErr_SetString(PyExc_OverflowError, "Python int too large to convert to i32"); return -1; } From 710ecb36bbd441e39cf203c5b70bfd500910dc03 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 13:20:56 +0000 Subject: [PATCH 204/208] Add module docstring --- mypyc/lib-rt/vecs/librt_vecs.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/librt_vecs.c b/mypyc/lib-rt/vecs/librt_vecs.c index 276c71c0133de..bbbefd90cf92b 100644 --- a/mypyc/lib-rt/vecs/librt_vecs.c +++ b/mypyc/lib-rt/vecs/librt_vecs.c @@ -903,7 +903,7 @@ static PyModuleDef_Slot librt_vecs_module_slots[] = { static PyModuleDef vecsmodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "vecs", - .m_doc = "vecs doc", + .m_doc = "This module implements the vec type and related functionality.", .m_size = 0, .m_methods = VecsMethods, .m_slots = librt_vecs_module_slots, From 80cefdd05959094e710ab1b16b60fe88f3d7ce68 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 13:29:37 +0000 Subject: [PATCH 205/208] Document macros in vec_template.c --- mypyc/lib-rt/vecs/vec_template.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index b490c4c1d6fa0..4d958e91ac55b 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -6,7 +6,20 @@ // as vec[i64] or vec[float]. Assume that certain #defines are provided that // provide all the item type specific definitions: // -// VEC vec C type (e.g. VecI64) +// VEC vec C type (e.g. VecI32) +// VEC_TYPE boxed vec type object (e.g. VecI32Type) +// VEC_OBJECT boxed Python object struct (e.g. VecI32Object) +// BUF_OBJECT buffer Python object struct (e.g. VecI32BufObject) +// BUF_TYPE buffer type object (e.g. VecI32BufType) +// NAME(suffix) macro to create prefixed name with given suffix (e.g. VecI32##suffix) +// FUNC(suffix) macro to create prefixed function name with suffix (e.g. VecI32_##suffix) +// ITEM_TYPE_STR vec item type as C string literal (e.g. "i32") +// ITEM_TYPE_MAGIC integer constant corresponding to the item type (e.g. VEC_ITEM_TYPE_I32) +// ITEM_C_TYPE C type used for items (e.g. int32_t) +// FEATURES capsule API struct name (e.g. Vec_I32API) +// BOX_ITEM C function to box item (e.g. VecI32_BoxItem) +// UNBOX_ITEM C function to unbox item (e.g. VecI32_UnboxItem) +// IS_UNBOX_ERROR C function to check for unbox error (e.g. VecI32_IsUnboxError) #ifndef VEC #error "VEC must be defined" From f0a6a54c12647fbc92236464f02fdac4b5aa3eed Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 15:45:19 +0000 Subject: [PATCH 206/208] Fix bad tp_basicsize for buf objects on 32-bit platforms at least --- mypyc/lib-rt/vecs/vec_template.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypyc/lib-rt/vecs/vec_template.c b/mypyc/lib-rt/vecs/vec_template.c index 4d958e91ac55b..864074cd10878 100644 --- a/mypyc/lib-rt/vecs/vec_template.c +++ b/mypyc/lib-rt/vecs/vec_template.c @@ -357,7 +357,7 @@ PyTypeObject BUF_TYPE = { PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "vecbuf[" ITEM_TYPE_STR "]", .tp_doc = "vec doc", - .tp_basicsize = sizeof(VEC_OBJECT) - sizeof(ITEM_C_TYPE), + .tp_basicsize = sizeof(BUF_OBJECT) - sizeof(ITEM_C_TYPE), .tp_itemsize = sizeof(ITEM_C_TYPE), .tp_flags = Py_TPFLAGS_DEFAULT, //.tp_new = ?? From 0a920532d4248346d7c145cdc24c4a1a73089dae Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 15:58:56 +0000 Subject: [PATCH 207/208] Work around failing test on 32-bit platforms --- mypyc/test-data/run-vecs-i64-interp.test | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index 86b60f7fd0335..f2add01c1d136 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -81,7 +81,8 @@ def test_repr() -> None: def test_size() -> None: v = vec[i64]() - if not is_gil_disabled(): + # Only test on 64-bit, non-free-threading builds + if not is_gil_disabled() and sys.maxsize > 2**32: assert getsizeof(v) == 32 v = append(v, 123) assert getsizeof(v) == 32 From 32e12b2607e4573278e7bca11b7e5badc15a0555 Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 27 Jan 2026 16:09:10 +0000 Subject: [PATCH 208/208] Work around issue with test stubs --- mypyc/test-data/fixtures/testutil.py | 4 ++++ mypyc/test-data/run-vecs-i64-interp.test | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mypyc/test-data/fixtures/testutil.py b/mypyc/test-data/fixtures/testutil.py index a0e0c0c04534e..60e99256c5671 100644 --- a/mypyc/test-data/fixtures/testutil.py +++ b/mypyc/test-data/fixtures/testutil.py @@ -106,3 +106,7 @@ def g(*args: Any, **kwargs: Any) -> Any: def is_gil_disabled() -> bool: return hasattr(sys, "_is_gil_enabled") and not sys._is_gil_enabled() + + +def is_64_bit_platform() -> bool: + return getattr(sys, "maxsize") > 2**32 diff --git a/mypyc/test-data/run-vecs-i64-interp.test b/mypyc/test-data/run-vecs-i64-interp.test index f2add01c1d136..a1dcde6219c1e 100644 --- a/mypyc/test-data/run-vecs-i64-interp.test +++ b/mypyc/test-data/run-vecs-i64-interp.test @@ -13,7 +13,7 @@ import typing import librt.vecs import mypy_extensions -from testutil import assertRaises, is_gil_disabled +from testutil import assertRaises, is_gil_disabled, is_64_bit_platform # Access via Any references to force generic operations vec: Any = librt.vecs.vec @@ -82,7 +82,7 @@ def test_repr() -> None: def test_size() -> None: v = vec[i64]() # Only test on 64-bit, non-free-threading builds - if not is_gil_disabled() and sys.maxsize > 2**32: + if not is_gil_disabled() and is_64_bit_platform(): assert getsizeof(v) == 32 v = append(v, 123) assert getsizeof(v) == 32