From 679c22d3a119c9688212e89891216cef7ed71726 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 25 Jun 2024 16:05:59 +0530 Subject: [PATCH 01/19] chore: modify VectorInit to handle numyp multi dimensional arrays --- src/Pythonize.cxx | 63 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 9dc78d2d..c18bdb5e 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -463,16 +463,16 @@ PyObject* VectorIAdd(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { -// Specialized vector constructor to allow construction from containers; allowing -// such construction from initializer_list instead would possible, but can be -// error-prone. This use case is common enough for std::vector to implement it -// directly, except for arrays (which can be passed wholesale) and strings (which -// won't convert properly as they'll be seen as buffers) + // Specialized vector constructor to allow construction from containers; allowing + // such construction from initializer_list instead would possible, but can be + // error-prone. This use case is common enough for std::vector to implement it + // directly, except for arrays (which can be passed wholesale) and strings (which + // won't convert properly as they'll be seen as buffers) ItemGetter* getter = GetGetter(args); if (getter) { - // construct an empty vector, then back-fill it + // construct an empty vector, then back-fill it PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); if (!result) { delete getter; @@ -490,7 +490,55 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return result; } -// The given argument wasn't iterable: simply forward to regular constructor + // Check if the argument is a numpy array + PyObject* numpy_module = PyImport_ImportModule("numpy"); + if (numpy_module) { + PyObject* numpy_array = PyObject_GetAttrString(numpy_module, "ndarray"); + if (numpy_array && PyArray_Check(args)) { + // Convert the numpy array to a vector + PyArrayObject* array = reinterpret_cast(args); + npy_intp size = PyArray_SIZE(array); + npy_intp* shape = PyArray_SHAPE(array); + npy_intp ndim = PyArray_NDIM(array); + npy_intp* strides = PyArray_STRIDES(array); + npy_intp itemsize = PyArray_ITEMSIZE(array); + void* data = PyArray_DATA(array); + + // Create a vector with the same type as the numpy array + PyObject* vector_type = PyObject_GetAttrString(self, "__class__"); + PyObject* vector = PyObject_CallFunctionObjArgs(vector_type, nullptr); + Py_DECREF(vector_type); + + // Resize the vector to match the shape of the numpy array + PyObject* resize_method = PyObject_GetAttrString(vector, "resize"); + PyObject* resize_args = PyTuple_New(ndim); + for (npy_intp i = 0; i < ndim; ++i) { + PyTuple_SetItem(resize_args, i, PyLong_FromLong(shape[i])); + } + PyObject* resize_result = PyObject_Call(resize_method, resize_args, nullptr); + Py_DECREF(resize_method); + Py_DECREF(resize_args); + Py_DECREF(resize_result); + + // Copy the data from the numpy array to the vector + PyObject* assign_method = PyObject_GetAttrString(vector, "assign"); + PyObject* assign_args = PyTuple_New(1); + PyTuple_SetItem(assign_args, 0, PyBytes_FromStringAndSize(static_cast(data), size * itemsize)); + PyObject* assign_result = PyObject_Call(assign_method, assign_args, nullptr); + Py_DECREF(assign_method); + Py_DECREF(assign_args); + Py_DECREF(assign_result); + + Py_DECREF(numpy_array); + Py_DECREF(numpy_module); + + return vector; + } + Py_DECREF(numpy_array); + Py_DECREF(numpy_module); + } + + // The given argument wasn't iterable or a numpy array: simply forward to regular constructor PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); if (realInit) { PyObject* result = PyObject_Call(realInit, args, nullptr); @@ -500,7 +548,6 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return nullptr; } - //--------------------------------------------------------------------------- PyObject* VectorData(PyObject* self, PyObject*) { From ae088a20e2f1ccfc8b28765c161a5781559b0815 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 2 Jul 2024 19:30:22 +0530 Subject: [PATCH 02/19] chore: add ndim-1 -- draft support --- src/Pythonize.cxx | 67 ++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 44 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index c18bdb5e..06fa1edf 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -490,52 +490,31 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return result; } - // Check if the argument is a numpy array - PyObject* numpy_module = PyImport_ImportModule("numpy"); - if (numpy_module) { - PyObject* numpy_array = PyObject_GetAttrString(numpy_module, "ndarray"); - if (numpy_array && PyArray_Check(args)) { - // Convert the numpy array to a vector - PyArrayObject* array = reinterpret_cast(args); - npy_intp size = PyArray_SIZE(array); - npy_intp* shape = PyArray_SHAPE(array); - npy_intp ndim = PyArray_NDIM(array); - npy_intp* strides = PyArray_STRIDES(array); - npy_intp itemsize = PyArray_ITEMSIZE(array); - void* data = PyArray_DATA(array); - - // Create a vector with the same type as the numpy array - PyObject* vector_type = PyObject_GetAttrString(self, "__class__"); - PyObject* vector = PyObject_CallFunctionObjArgs(vector_type, nullptr); - Py_DECREF(vector_type); - - // Resize the vector to match the shape of the numpy array - PyObject* resize_method = PyObject_GetAttrString(vector, "resize"); - PyObject* resize_args = PyTuple_New(ndim); - for (npy_intp i = 0; i < ndim; ++i) { - PyTuple_SetItem(resize_args, i, PyLong_FromLong(shape[i])); +// Return 1 if obj supports the buffer interface otherwise 0. + if (PyObject_CheckBuffer(args)){ + PyObject *mview = PyMemoryView_FromObject(args); + Py_buffer *buf = PyMemoryView_GET_BUFFER(mview); + + Py_buffer *array_buf = PyObject_GetBuffer(args, buf, PyBUF_FULL); + + if(!array_buf){ + PyErr_SetString(PyExc_TypeError, "argument is not iterable"); + return nullptr; + } + + int itemsize = array_buf->itemsize; + int len = array_buf->len; + int ndim = array_buf->ndim; + + if (ndim == 1 ){ + PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result){ + PyBuffer_Release(array_buf); + return nullptr; } - PyObject* resize_result = PyObject_Call(resize_method, resize_args, nullptr); - Py_DECREF(resize_method); - Py_DECREF(resize_args); - Py_DECREF(resize_result); - - // Copy the data from the numpy array to the vector - PyObject* assign_method = PyObject_GetAttrString(vector, "assign"); - PyObject* assign_args = PyTuple_New(1); - PyTuple_SetItem(assign_args, 0, PyBytes_FromStringAndSize(static_cast(data), size * itemsize)); - PyObject* assign_result = PyObject_Call(assign_method, assign_args, nullptr); - Py_DECREF(assign_method); - Py_DECREF(assign_args); - Py_DECREF(assign_result); - - Py_DECREF(numpy_array); - Py_DECREF(numpy_module); - - return vector; } - Py_DECREF(numpy_array); - Py_DECREF(numpy_module); + + PyBuffer_Release(array_buf); } // The given argument wasn't iterable or a numpy array: simply forward to regular constructor From 2b52c7edeae620635ee71eadafcdbba95b94f47e Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 16 Jul 2024 19:59:59 +0530 Subject: [PATCH 03/19] refactor: type checks and errors messages Code doesn't execute due to `PyObject_GetBuffer` non-matching function call, even though it is correct way as per the numpy buffer protocol docs --- src/Pythonize.cxx | 57 +++++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 06fa1edf..20acb646 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -13,6 +13,10 @@ #include "TypeManip.h" #include "Utility.h" +// Numpy headers +#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION +#include "numpy/ndarrayobject.h" + // Standard #include #include @@ -490,37 +494,46 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return result; } -// Return 1 if obj supports the buffer interface otherwise 0. - if (PyObject_CheckBuffer(args)){ - PyObject *mview = PyMemoryView_FromObject(args); - Py_buffer *buf = PyMemoryView_GET_BUFFER(mview); + PyObject *fi = PyTuple_GET_ITEM(args, 0); - Py_buffer *array_buf = PyObject_GetBuffer(args, buf, PyBUF_FULL); + // check if numpy is passed + if (PyObject_CheckBuffer(fi)){ - if(!array_buf){ - PyErr_SetString(PyExc_TypeError, "argument is not iterable"); - return nullptr; - } + PyObject *base = args; + Py_buffer *view = NULL; + + bool type_check = (CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi)); - int itemsize = array_buf->itemsize; - int len = array_buf->len; - int ndim = array_buf->ndim; - - if (ndim == 1 ){ - PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - if (!result){ - PyBuffer_Release(array_buf); + // Return 1 if obj supports the buffer interface otherwise 0. + if (!type_check){ + if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0) + PyErr_Clear(); + if (PyObject_GetBuffer(base, &view, PyBUF_SIMPLE) < 0) return nullptr; + + PyObject *vend = PyObject_GetAttr(self, PyStrings::gRealInit); + + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); + if (!vend) + PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); + + if (vend) + { + PyObject *result = PyObject_Call(vend, args, nullptr); + Py_DECREF(vend); + return result; } } - - PyBuffer_Release(array_buf); } + if (!PyErr_Occurred()) + PyErr_SetString(PyExc_TypeError, "argument is not iterable"); // The given argument wasn't iterable or a numpy array: simply forward to regular constructor - PyObject* realInit = PyObject_GetAttr(self, PyStrings::gRealInit); - if (realInit) { - PyObject* result = PyObject_Call(realInit, args, nullptr); + PyObject *realInit = PyObject_GetAttr(self, PyStrings::gRealInit); + if (realInit) + { + PyObject *result = PyObject_Call(realInit, args, nullptr); Py_DECREF(realInit); return result; } From f61220b89d9449f51caaf5e903348c7cb9a48a64 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 16 Jul 2024 20:30:40 +0530 Subject: [PATCH 04/19] fix: non-matching function call Code executes but Pycode return an error to handle TemplateProxy --- CMakeLists.txt | 1 + src/Pythonize.cxx | 32 +++++++++++++++----------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 272fcb33..910b2bcc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ if (NOT Python_Development_FOUND) endif() include_directories("${PROJECT_SOURCE_DIR}/include" "${Python_INCLUDE_DIRS}") +include_directories("/Users/khushiyant/Desktop/Development/gsoc/.venv/lib/python3.10/site-packages/numpy/_core/include") file (GLOB cppyy_src src/*.cxx) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 20acb646..746fafef 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -499,33 +499,31 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) // check if numpy is passed if (PyObject_CheckBuffer(fi)){ - PyObject *base = args; - Py_buffer *view = NULL; - + PyObject *base = PyObject_GetAttr(self, PyStrings::gRealInit); + PyObject *memoryview = PyMemoryView_FromObject(base); + + Py_buffer *view = PyMemoryView_GET_BUFFER(memoryview); + bool type_check = (CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi)); - + // Return 1 if obj supports the buffer interface otherwise 0. + if (!base) + PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); + if (!type_check){ - if (PyObject_GetBuffer(base, &view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0) + if (PyObject_GetBuffer(base, view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0){ PyErr_Clear(); - if (PyObject_GetBuffer(base, &view, PyBUF_SIMPLE) < 0) - return nullptr; - - PyObject *vend = PyObject_GetAttr(self, PyStrings::gRealInit); + if (PyObject_GetBuffer(base, view, PyBUF_SIMPLE) < 0) + return nullptr; + } if (!PyErr_Occurred()) PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); - if (!vend) - PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); - if (vend) - { - PyObject *result = PyObject_Call(vend, args, nullptr); - Py_DECREF(vend); - return result; - } + PyBuffer_Release(view); } } + if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "argument is not iterable"); From 2a09e3805709eec2062c4391280b7debbe51f94a Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 16 Jul 2024 20:56:09 +0530 Subject: [PATCH 05/19] refactor: code logic--loop Logic to implement the initialisation still remains --- src/Pythonize.cxx | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 746fafef..7f184afa 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -499,27 +499,41 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) // check if numpy is passed if (PyObject_CheckBuffer(fi)){ - PyObject *base = PyObject_GetAttr(self, PyStrings::gRealInit); - PyObject *memoryview = PyMemoryView_FromObject(base); + PyObject* base = PyObject_GetAttr(self, PyStrings::gRealInit); + PyObject* memoryview = PyMemoryView_FromObject(base); + Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); - Py_buffer *view = PyMemoryView_GET_BUFFER(memoryview); - - bool type_check = (CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi)); + int dataflags = NPY_ARRAY_BEHAVED; // Return 1 if obj supports the buffer interface otherwise 0. if (!base) PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); - if (!type_check){ + if (!(CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi))) + { if (PyObject_GetBuffer(base, view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0){ PyErr_Clear(); if (PyObject_GetBuffer(base, view, PyBUF_SIMPLE) < 0) return nullptr; + dataflags &= ~NPY_ARRAY_WRITEABLE; } if (!PyErr_Occurred()) PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); + + + + // logic to return the PyObject for numpy ndarrays + PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + + + int fillsz = view->len; + for (Py_ssize_t i = 0; i < fillsz; ++i){ + continue; + } + + // dereference the memoryview buffer PyBuffer_Release(view); } } From c95571ac4e921c6855585c88505d0d442002c5df Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 16 Jul 2024 21:17:03 +0530 Subject: [PATCH 06/19] chore: add 2 approaches as comments --- src/Pythonize.cxx | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 7f184afa..b17818c2 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -522,16 +522,44 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); - - // logic to return the PyObject for numpy ndarrays - PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); - - int fillsz = view->len; - for (Py_ssize_t i = 0; i < fillsz; ++i){ - continue; - } + // Approach 1: + // PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + + // int fillsz = view->len; + // for (Py_ssize_t i = 0; i < fillsz; ++i){ + // PyObject* item = PySequence_GetItem(items, i); + // PyObject *index = PyInt_FromSsize_t(i); + // PyObject *sires = PyObject_CallFunctionObjArgs(si_call, index, item, nullptr); + // Py_DECREF(index); + // Py_DECREF(item); + // if (!sires) + // { + // Py_DECREF(si_call); + // Py_DECREF(result); + // return nullptr; + // } + // else + // Py_DECREF(sires); + // } + // Py_DECREF(si_call); + + // return result; + // } + + // Approach 2: + // PyArrayObject *ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( + // &PyArray_Type, dtype, + // n, dims, NULL, data, + // dataflags, NULL, base); + + // if (n) + // memcpy(PyArray_STRIDES(ret), strides, n * sizeof(npy_intp)); + + + // Py_DECREF(base); + // return (PyObject *)ret; // dereference the memoryview buffer PyBuffer_Release(view); From 2e6e14a4f13f858dbaea90f2b1a399925c202c6a Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Mon, 22 Jul 2024 19:01:38 +0530 Subject: [PATCH 07/19] chore: implemented approach similar to arrayinit It results in TypeError: a bytes-like object is required, not 'cppyy.TemplateProxy' --- src/Pythonize.cxx | 69 +++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index b17818c2..11167dd0 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -503,8 +503,6 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* memoryview = PyMemoryView_FromObject(base); Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); - int dataflags = NPY_ARRAY_BEHAVED; - // Return 1 if obj supports the buffer interface otherwise 0. if (!base) PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); @@ -513,58 +511,45 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { if (PyObject_GetBuffer(base, view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0){ PyErr_Clear(); - if (PyObject_GetBuffer(base, view, PyBUF_SIMPLE) < 0) + if (PyObject_GetBuffer(base, view, PyBUF_SIMPLE) < 0){ return nullptr; - dataflags &= ~NPY_ARRAY_WRITEABLE; + } } - if (!PyErr_Occurred()) + if (!PyErr_Occurred()){ PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); - + } // logic to return the PyObject for numpy ndarrays // Approach 1: - // PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); - - // int fillsz = view->len; - // for (Py_ssize_t i = 0; i < fillsz; ++i){ - // PyObject* item = PySequence_GetItem(items, i); - // PyObject *index = PyInt_FromSsize_t(i); - // PyObject *sires = PyObject_CallFunctionObjArgs(si_call, index, item, nullptr); - // Py_DECREF(index); - // Py_DECREF(item); - // if (!sires) - // { - // Py_DECREF(si_call); - // Py_DECREF(result); - // return nullptr; - // } - // else - // Py_DECREF(sires); - // } - // Py_DECREF(si_call); - - // return result; - // } - - // Approach 2: - // PyArrayObject *ret = (PyArrayObject *)PyArray_NewFromDescrAndBase( - // &PyArray_Type, dtype, - // n, dims, NULL, data, - // dataflags, NULL, base); - - // if (n) - // memcpy(PyArray_STRIDES(ret), strides, n * sizeof(npy_intp)); - - - // Py_DECREF(base); - // return (PyObject *)ret; + PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); + + int fillsz = view->len; + + for (Py_ssize_t i = 0; i < fillsz; ++i){ + PyObject* item = PySequence_GetItem(fi, i); + PyObject *index = PyInt_FromSsize_t(i); + PyObject *sires = PyObject_CallFunctionObjArgs(si_call, index, item, nullptr); + Py_DECREF(index); + Py_DECREF(item); + if (!sires){ + Py_DECREF(si_call); + Py_DECREF(base); + return nullptr; + } + else + Py_DECREF(sires); + } + Py_DECREF(si_call); + + return base; + } // dereference the memoryview buffer PyBuffer_Release(view); } - } + if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "argument is not iterable"); From 0cc4d6c0aa1e020f03ba8a76248906f3ba4063c8 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 23 Jul 2024 17:38:38 +0530 Subject: [PATCH 08/19] chore: add iter--handling for nested arrays --- src/Pythonize.cxx | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 11167dd0..41333437 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -521,35 +521,36 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } // logic to return the PyObject for numpy ndarrays - - // Approach 1: PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); int fillsz = view->len; for (Py_ssize_t i = 0; i < fillsz; ++i){ PyObject* item = PySequence_GetItem(fi, i); - PyObject *index = PyInt_FromSsize_t(i); - PyObject *sires = PyObject_CallFunctionObjArgs(si_call, index, item, nullptr); - Py_DECREF(index); - Py_DECREF(item); - if (!sires){ - Py_DECREF(si_call); - Py_DECREF(base); + ItemGetter *getter = GetGetter(item); + + if (getter){ + // construct an empty vector, then back-fill it + PyObject *result = PyObject_CallMethodNoArgs(item, PyStrings::gRealInit); + if (!result){ + delete getter; return nullptr; } - else - Py_DECREF(sires); - } - Py_DECREF(si_call); - return base; - } + bool fill_ok = FillVector(self, args, getter); + delete getter; + if (!fill_ok){ + Py_DECREF(result); + return nullptr; + } + return result; + } + } // dereference the memoryview buffer PyBuffer_Release(view); + } - if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "argument is not iterable"); @@ -565,6 +566,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return nullptr; } +} //--------------------------------------------------------------------------- PyObject* VectorData(PyObject* self, PyObject*) { From 6416c498ebec8710a2f17e5e3f2f1bbd30d96d1f Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 23 Jul 2024 17:59:09 +0530 Subject: [PATCH 09/19] chore: remove unused--si_call var --- src/Pythonize.cxx | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 41333437..8aa7023e 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -13,10 +13,6 @@ #include "TypeManip.h" #include "Utility.h" -// Numpy headers -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include "numpy/ndarrayobject.h" - // Standard #include #include @@ -521,8 +517,6 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } // logic to return the PyObject for numpy ndarrays - PyObject *si_call = PyObject_GetAttr(self, PyStrings::gSetItem); - int fillsz = view->len; for (Py_ssize_t i = 0; i < fillsz; ++i){ @@ -549,8 +543,8 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } // dereference the memoryview buffer PyBuffer_Release(view); - } + } if (!PyErr_Occurred()) PyErr_SetString(PyExc_TypeError, "argument is not iterable"); @@ -566,7 +560,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return nullptr; } -} + //--------------------------------------------------------------------------- PyObject* VectorData(PyObject* self, PyObject*) { From 357c15ec7f2d43e4a224baec7a11d76ad17b0aae Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 23 Jul 2024 21:12:15 +0530 Subject: [PATCH 10/19] chore: update pyobject method to __array__ --- src/Pythonize.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 8aa7023e..f02bee51 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -525,7 +525,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) if (getter){ // construct an empty vector, then back-fill it - PyObject *result = PyObject_CallMethodNoArgs(item, PyStrings::gRealInit); + PyObject *result = CallPyObjMethod(item, "__array__"); if (!result){ delete getter; return nullptr; From c45bef3458ffab1dea8659151ba521f70972c50c Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 30 Jul 2024 13:54:29 +0530 Subject: [PATCH 11/19] chore: add 1d logic --- src/Pythonize.cxx | 101 ++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index f02bee51..67c133bc 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -473,7 +473,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) if (getter) { // construct an empty vector, then back-fill it - PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gArray); if (!result) { delete getter; return nullptr; @@ -490,64 +490,67 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return result; } - PyObject *fi = PyTuple_GET_ITEM(args, 0); + PyObject* fi = PyTuple_GET_ITEM(args, 0); + if (fi == Py_None) { + // empty vector + return PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + + } // check if numpy is passed if (PyObject_CheckBuffer(fi)){ - PyObject* base = PyObject_GetAttr(self, PyStrings::gRealInit); - PyObject* memoryview = PyMemoryView_FromObject(base); + PyObject* memoryview = PyMemoryView_FromObject(fi); Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); - // Return 1 if obj supports the buffer interface otherwise 0. - if (!base) - PyErr_SetString(PyExc_BufferError, "attempt to access a null-pointer"); - - if (!(CPyCppyy_PyText_Check(fi) || PyBytes_Check(fi))) - { - if (PyObject_GetBuffer(base, view, PyBUF_WRITABLE | PyBUF_SIMPLE) < 0){ - PyErr_Clear(); - if (PyObject_GetBuffer(base, view, PyBUF_SIMPLE) < 0){ + if (!view) + return nullptr; + + // logic to return the PyObject for numpy ndarrays + if (view->ndim == 1){ + // Create a new vector object + PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!vector_obj) + { + return nullptr; + } + Py_ssize_t fillsz = view->len / view->itemsize; + std::vector vec(fillsz); + + PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); + for (Py_ssize_t i = 0; i < fillsz; i++) + { + double *val = (double *)((char *)view->buf + i * view->itemsize); + PyObject *item = PyFloat_FromDouble(*val); + if (!item) + { + Py_DECREF(vector_obj); return nullptr; - } } - - if (!PyErr_Occurred()){ - PyErr_SetString(PyExc_BufferError, "exporter cannot provide a buffer of the exact type"); - } - - // logic to return the PyObject for numpy ndarrays - int fillsz = view->len; - - for (Py_ssize_t i = 0; i < fillsz; ++i){ - PyObject* item = PySequence_GetItem(fi, i); - ItemGetter *getter = GetGetter(item); - - if (getter){ - // construct an empty vector, then back-fill it - PyObject *result = CallPyObjMethod(item, "__array__"); - if (!result){ - delete getter; - return nullptr; - } - - bool fill_ok = FillVector(self, args, getter); - delete getter; - - if (!fill_ok){ - Py_DECREF(result); - return nullptr; + PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); + if (!pbres) + { + Py_DECREF(vector_obj); + break; + return nullptr; } - return result; - } - } - // dereference the memoryview buffer - PyBuffer_Release(view); + Py_DECREF(item); + } + return vector_obj; + } + else + { + // logic for ND + return nullptr; } - } - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, "argument is not iterable"); + // dereference the memoryview buffer + Py_DECREF(memoryview); + PyBuffer_Release(view); +} +Py_DECREF(fi); + + // The given argument wasn't iterable or a numpy array: simply forward to regular constructor PyObject *realInit = PyObject_GetAttr(self, PyStrings::gRealInit); @@ -1856,8 +1859,8 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) Utility::AddToClass(pyclass, "__setitem__", (PyCFunction)VectorBoolSetItem); } else { // constructor that takes python collections - Utility::AddToClass(pyclass, "__real_init", "__init__"); Utility::AddToClass(pyclass, "__init__", (PyCFunction)VectorInit, METH_VARARGS | METH_KEYWORDS); + Utility::AddToClass(pyclass, "__real_init", "__init__"); // data with size Utility::AddToClass(pyclass, "__real_data", "data"); From 2fd19a2af9df4342fe51bf369f0229fabebd766e Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 30 Jul 2024 13:59:09 +0530 Subject: [PATCH 12/19] chore: reformmating code sample for readabilty --- src/Pythonize.cxx | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 67c133bc..97a5bf3f 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -494,7 +494,6 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) if (fi == Py_None) { // empty vector return PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - } // check if numpy is passed @@ -503,48 +502,50 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* memoryview = PyMemoryView_FromObject(fi); Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); - if (!view) - return nullptr; + if (!view) return nullptr; // logic to return the PyObject for numpy ndarrays if (view->ndim == 1){ // Create a new vector object PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - if (!vector_obj) - { - return nullptr; - } + if (!vector_obj) return nullptr; + + // number of elements Py_ssize_t fillsz = view->len / view->itemsize; std::vector vec(fillsz); + // push_back attribute PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); - for (Py_ssize_t i = 0; i < fillsz; i++) - { + + for (Py_ssize_t i = 0; i < fillsz; i++) { + + // accessing item for buffer double *val = (double *)((char *)view->buf + i * view->itemsize); PyObject *item = PyFloat_FromDouble(*val); - if (!item) - { + + if (!item) { Py_DECREF(vector_obj); return nullptr; } + // element push back PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); - if (!pbres) - { + + if (!pbres) { Py_DECREF(vector_obj); break; return nullptr; - } + } + Py_DECREF(item); } + return vector_obj; - } - else - { + } else { // logic for ND return nullptr; } - // dereference the memoryview buffer + // dereference the memoryview buffer Py_DECREF(memoryview); PyBuffer_Release(view); } From 363b864883180987d6c58738d1b76c008c5e74f4 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 30 Jul 2024 14:39:13 +0530 Subject: [PATCH 13/19] chore: add logic for nd arrays using recurssive call --- src/Pythonize.cxx | 68 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 97a5bf3f..b8f108a3 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -541,15 +541,73 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return vector_obj; } else { - // logic for ND - return nullptr; - } + // Handle multi-dimensional array case + PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!vector_obj) + return nullptr; + + Py_ssize_t size = view->shape[0]; + for (Py_ssize_t i = 0; i < size; i++) + { + // Create a new memory view for the sub-array + Py_buffer sub_view; + sub_view.buf = (char *)view->buf + i * view->strides[0]; + sub_view.len = view->len - i * view->strides[0]; + sub_view.ndim = view->ndim - 1; + sub_view.shape = view->shape + 1; + sub_view.strides = view->strides + 1; + sub_view.suboffsets = view->suboffsets + 1; + sub_view.readonly = view->readonly; + + // Create a new PyObject for the sub-array + PyObject *sub_array = PyMemoryView_FromBuffer(&sub_view); + if (!sub_array) + { + Py_DECREF(vector_obj); + return nullptr; + } + + // Recursively call VectorInit on the sub-array + PyObject *sub_vector = VectorInit(self, PyTuple_Pack(1, sub_array), nullptr); + if (!sub_vector) + { + Py_DECREF(vector_obj); + Py_DECREF(sub_array); + return nullptr; + } + + // Push the sub-vector onto the main vector + PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); + if (!pb_call) + { + Py_DECREF(vector_obj); + Py_DECREF(sub_array); + Py_DECREF(sub_vector); + return nullptr; + } + + PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, sub_vector, nullptr); + if (!pbres) + { + Py_DECREF(vector_obj); + Py_DECREF(sub_array); + Py_DECREF(sub_vector); + Py_DECREF(pb_call); + return nullptr; + } + Py_DECREF(sub_array); + Py_DECREF(sub_vector); + Py_DECREF(pb_call); + } + + return vector_obj; + } // dereference the memoryview buffer Py_DECREF(memoryview); PyBuffer_Release(view); -} -Py_DECREF(fi); + } + Py_DECREF(fi); From 2d5f981f4dba0a7f2dceba92769caac0a31cdc33 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 30 Jul 2024 14:47:12 +0530 Subject: [PATCH 14/19] chore: remove numpy core directories --- CMakeLists.txt | 1 - src/Pythonize.cxx | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 910b2bcc..272fcb33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,6 @@ if (NOT Python_Development_FOUND) endif() include_directories("${PROJECT_SOURCE_DIR}/include" "${Python_INCLUDE_DIRS}") -include_directories("/Users/khushiyant/Desktop/Development/gsoc/.venv/lib/python3.10/site-packages/numpy/_core/include") file (GLOB cppyy_src src/*.cxx) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index b8f108a3..951d8343 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -463,17 +463,17 @@ PyObject* VectorIAdd(PyObject* self, PyObject* args, PyObject* /* kwds */) PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { - // Specialized vector constructor to allow construction from containers; allowing - // such construction from initializer_list instead would possible, but can be - // error-prone. This use case is common enough for std::vector to implement it - // directly, except for arrays (which can be passed wholesale) and strings (which - // won't convert properly as they'll be seen as buffers) +// Specialized vector constructor to allow construction from containers; allowing +// such construction from initializer_list instead would possible, but can be +// error-prone. This use case is common enough for std::vector to implement it +// directly, except for arrays (which can be passed wholesale) and strings (which +// won't convert properly as they'll be seen as buffers) ItemGetter* getter = GetGetter(args); if (getter) { - // construct an empty vector, then back-fill it - PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gArray); + // construct an empty vector, then back-fill it + PyObject* result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); if (!result) { delete getter; return nullptr; @@ -1918,8 +1918,8 @@ bool CPyCppyy::Pythonize(PyObject* pyclass, const std::string& name) Utility::AddToClass(pyclass, "__setitem__", (PyCFunction)VectorBoolSetItem); } else { // constructor that takes python collections - Utility::AddToClass(pyclass, "__init__", (PyCFunction)VectorInit, METH_VARARGS | METH_KEYWORDS); Utility::AddToClass(pyclass, "__real_init", "__init__"); + Utility::AddToClass(pyclass, "__init__", (PyCFunction)VectorInit, METH_VARARGS | METH_KEYWORDS); // data with size Utility::AddToClass(pyclass, "__real_data", "data"); From 3ac1fdf9e25588bdb0ea25fda0b4c14aa069b1f7 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Wed, 14 Aug 2024 20:31:29 +0530 Subject: [PATCH 15/19] feat: 1D numpy array supprt --- src/Pythonize.cxx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 951d8343..a8e98bb1 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -507,31 +507,30 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) // logic to return the PyObject for numpy ndarrays if (view->ndim == 1){ // Create a new vector object - PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - if (!vector_obj) return nullptr; + PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + if (!result) return nullptr; // number of elements Py_ssize_t fillsz = view->len / view->itemsize; - std::vector vec(fillsz); // push_back attribute - PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); + PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); for (Py_ssize_t i = 0; i < fillsz; i++) { // accessing item for buffer double *val = (double *)((char *)view->buf + i * view->itemsize); - PyObject *item = PyFloat_FromDouble(*val); + PyObject *item = PyLong_FromLong(*val); if (!item) { - Py_DECREF(vector_obj); + Py_DECREF(result); return nullptr; } // element push back PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); if (!pbres) { - Py_DECREF(vector_obj); + Py_DECREF(result); break; return nullptr; } @@ -539,7 +538,7 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) Py_DECREF(item); } - return vector_obj; + return result; } else { // Handle multi-dimensional array case PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); From 873bcd2fc66a05b91eb68284f0ea7f41d4a4b5df Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Thu, 15 Aug 2024 19:09:49 +0530 Subject: [PATCH 16/19] refactor: seperate the index value logic --- src/Pythonize.cxx | 60 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index a8e98bb1..e7ed8c1d 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -51,6 +51,34 @@ bool HasAttrDirect(PyObject* pyclass, PyObject* pyname, bool mustBeCPyCppyy = fa PyErr_Clear(); return false; } +double Get_IndexValue(Py_buffer *view, Py_ssize_t i) +{ + // Check if the index is within bounds + if (i < 0 || i >= view->len / view->itemsize) + { + fprintf(stderr, "Index out of bounds.\n"); + return 0.0; // Return an error value or handle as appropriate + } + + // Calculate the pointer to the i-th element + char *element_ptr; + + if (view->strides == NULL) + { + // For contiguous arrays, compute the address directly + element_ptr = (char *)view->buf + i * view->itemsize; + } + else + { + // For non-contiguous arrays, use the stride + element_ptr = (char *)view->buf + i * view->strides[0]; + } + + // Cast the pointer to the appropriate type (double in this case) + double value = *(double *)element_ptr; + + return value; +} PyObject* GetAttrDirect(PyObject* pyclass, PyObject* pyname) { // get an attribute without causing getattr lookups @@ -497,12 +525,14 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) } // check if numpy is passed - if (PyObject_CheckBuffer(fi)){ + if (PyObject_CheckBuffer(fi)){ PyObject* memoryview = PyMemoryView_FromObject(fi); Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); - if (!view) return nullptr; + if (view->buf == NULL) return nullptr; + + // print elements of buffer here // logic to return the PyObject for numpy ndarrays if (view->ndim == 1){ @@ -515,12 +545,12 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) // push_back attribute PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); - + for (Py_ssize_t i = 0; i < fillsz; i++) { // accessing item for buffer - double *val = (double *)((char *)view->buf + i * view->itemsize); - PyObject *item = PyLong_FromLong(*val); + double val = Get_IndexValue(view, i); + PyObject *item = PyLong_FromDouble(val); if (!item) { Py_DECREF(result); @@ -549,14 +579,14 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) for (Py_ssize_t i = 0; i < size; i++) { // Create a new memory view for the sub-array - Py_buffer sub_view; + Py_buffer sub_view; sub_view.buf = (char *)view->buf + i * view->strides[0]; sub_view.len = view->len - i * view->strides[0]; sub_view.ndim = view->ndim - 1; sub_view.shape = view->shape + 1; sub_view.strides = view->strides + 1; sub_view.suboffsets = view->suboffsets + 1; - sub_view.readonly = view->readonly; + sub_view.readonly = view->readonly; // Create a new PyObject for the sub-array PyObject *sub_array = PyMemoryView_FromBuffer(&sub_view); @@ -564,16 +594,16 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { Py_DECREF(vector_obj); return nullptr; - } + } - // Recursively call VectorInit on the sub-array + // Recursively call VectorInit on the sub-array PyObject *sub_vector = VectorInit(self, PyTuple_Pack(1, sub_array), nullptr); if (!sub_vector) { Py_DECREF(vector_obj); Py_DECREF(sub_array); - return nullptr; - } + return nullptr; + } // Push the sub-vector onto the main vector PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); @@ -593,19 +623,19 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) Py_DECREF(sub_vector); Py_DECREF(pb_call); return nullptr; - } + } Py_DECREF(sub_array); Py_DECREF(sub_vector); Py_DECREF(pb_call); - } + } return vector_obj; - } +} // dereference the memoryview buffer Py_DECREF(memoryview); PyBuffer_Release(view); - } + } Py_DECREF(fi); From 950fea24c9bb3f25081bd69957cfec882c830974 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Thu, 15 Aug 2024 23:21:42 +0530 Subject: [PATCH 17/19] feat: add support for n-d numpy array --- src/Pythonize.cxx | 141 +++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 70 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index e7ed8c1d..45cdf82c 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -518,48 +518,52 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) return result; } + // get the first argument PyObject* fi = PyTuple_GET_ITEM(args, 0); - if (fi == Py_None) { + if (fi == Py_None) + { // empty vector return PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - } + } // check if numpy is passed - if (PyObject_CheckBuffer(fi)){ + // create a memoryview PyObject* memoryview = PyMemoryView_FromObject(fi); Py_buffer* view = PyMemoryView_GET_BUFFER(memoryview); + // check if memoryview is valid if (view->buf == NULL) return nullptr; - // print elements of buffer here - - // logic to return the PyObject for numpy ndarrays - if (view->ndim == 1){ + // logic to handle 1-dim buffer (i.e. numpy array) + if (view->ndim == 1) + { // Create a new vector object PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); if (!result) return nullptr; - // number of elements + // number of elements in the buffer Py_ssize_t fillsz = view->len / view->itemsize; // push_back attribute PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); - for (Py_ssize_t i = 0; i < fillsz; i++) { - + for (Py_ssize_t i = 0; i < fillsz; i++) + { // accessing item for buffer double val = Get_IndexValue(view, i); PyObject *item = PyLong_FromDouble(val); - if (!item) { + if (!item) + { Py_DECREF(result); return nullptr; } // element push back PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); - if (!pbres) { + if (!pbres) + { Py_DECREF(result); break; return nullptr; @@ -568,75 +572,72 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) Py_DECREF(item); } + // dereference the memoryview buffer + Py_DECREF(memoryview); + PyBuffer_Release(view); + Py_DECREF(fi); + return result; - } else { - // Handle multi-dimensional array case - PyObject *vector_obj = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - if (!vector_obj) - return nullptr; + } + // logic to handle n-dim buffer (i.e. numpy array of arrays) + else + { + PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); + PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); - Py_ssize_t size = view->shape[0]; - for (Py_ssize_t i = 0; i < size; i++) + if (!result) { - // Create a new memory view for the sub-array - Py_buffer sub_view; - sub_view.buf = (char *)view->buf + i * view->strides[0]; - sub_view.len = view->len - i * view->strides[0]; - sub_view.ndim = view->ndim - 1; - sub_view.shape = view->shape + 1; - sub_view.strides = view->strides + 1; - sub_view.suboffsets = view->suboffsets + 1; - sub_view.readonly = view->readonly; - - // Create a new PyObject for the sub-array - PyObject *sub_array = PyMemoryView_FromBuffer(&sub_view); - if (!sub_array) - { - Py_DECREF(vector_obj); - return nullptr; - } + Py_DECREF(memoryview); + PyBuffer_Release(view); + Py_DECREF(fi); + return nullptr; + } - // Recursively call VectorInit on the sub-array - PyObject *sub_vector = VectorInit(self, PyTuple_Pack(1, sub_array), nullptr); - if (!sub_vector) - { - Py_DECREF(vector_obj); - Py_DECREF(sub_array); - return nullptr; - } + for (Py_ssize_t i = 0; i < view->ndim; i++) { + // creating a tmp list for each dimension to pass to push_back + PyObject *item_vec = PyList_New(view->shape[1]); - // Push the sub-vector onto the main vector - PyObject *pb_call = PyObject_GetAttrString(vector_obj, (char *)"push_back"); - if (!pb_call) - { - Py_DECREF(vector_obj); - Py_DECREF(sub_array); - Py_DECREF(sub_vector); + if (!item_vec) { + Py_DECREF(result); + Py_DECREF(memoryview); + PyBuffer_Release(view); + Py_DECREF(fi); return nullptr; } - PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, sub_vector, nullptr); - if (!pbres) - { - Py_DECREF(vector_obj); - Py_DECREF(sub_array); - Py_DECREF(sub_vector); - Py_DECREF(pb_call); - return nullptr; - } + // accessing item for buffer + for (Py_ssize_t j = 0; j < view->shape[1]; j++) { - Py_DECREF(sub_array); - Py_DECREF(sub_vector); - Py_DECREF(pb_call); - } + int idx = i * view->shape[1] + j; + double val = Get_IndexValue(view, idx); + PyObject *item = PyLong_FromDouble(val); - return vector_obj; -} - // dereference the memoryview buffer - Py_DECREF(memoryview); - PyBuffer_Release(view); + if (!item) { + Py_DECREF(result); + Py_DECREF(item_vec); + return nullptr; + } + PyList_SetItem(item_vec, j, item); + Py_DECREF(item); + } + + // element push back + PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item_vec, nullptr); + if (!pbres) { + Py_DECREF(result); + Py_DECREF(item_vec); + break; + return nullptr; + } + } + // dereference the memoryview buffer + Py_DECREF(memoryview); + PyBuffer_Release(view); + Py_DECREF(fi); + + return result; } - Py_DECREF(fi); + } From b21587b25d9f8204ed808fa175f1501e7b53ca24 Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Sat, 17 Aug 2024 21:58:27 +0530 Subject: [PATCH 18/19] refactor: seperate out logic for recursion --- src/Pythonize.cxx | 239 ++++++++++++++++++++++------------------------ 1 file changed, 115 insertions(+), 124 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 45cdf82c..8e6c22e9 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -51,33 +51,23 @@ bool HasAttrDirect(PyObject* pyclass, PyObject* pyname, bool mustBeCPyCppyy = fa PyErr_Clear(); return false; } -double Get_IndexValue(Py_buffer *view, Py_ssize_t i) + +template +T Get_IndexValue(Py_buffer *view, int i) { - // Check if the index is within bounds - if (i < 0 || i >= view->len / view->itemsize) + if (!view || !view->buf) { - fprintf(stderr, "Index out of bounds.\n"); - return 0.0; // Return an error value or handle as appropriate + // Handle the error, e.g., throw an exception or return a default value + PyErr_SetString(PyExc_RuntimeError, "Buffer is not valid"); } - // Calculate the pointer to the i-th element - char *element_ptr; - - if (view->strides == NULL) - { - // For contiguous arrays, compute the address directly - element_ptr = (char *)view->buf + i * view->itemsize; - } - else + if (i < 0 || i >= view->len / view->itemsize) { - // For non-contiguous arrays, use the stride - element_ptr = (char *)view->buf + i * view->strides[0]; + // Handle the error, e.g., throw an exception or return a default value + PyErr_SetString(PyExc_IndexError, "Index out of range"); } - - // Cast the pointer to the appropriate type (double in this case) - double value = *(double *)element_ptr; - - return value; + // get the value at index i in the view + return *(T *)((char *)view->buf + i * view->strides[0]); } PyObject* GetAttrDirect(PyObject* pyclass, PyObject* pyname) { @@ -487,7 +477,101 @@ PyObject* VectorIAdd(PyObject* self, PyObject* args, PyObject* /* kwds */) PyErr_SetString(PyExc_TypeError, "argument is not iterable"); return nullptr; // error already set } +PyObject *recursive_vector_init(PyObject *self, Py_buffer *view, PyObject *result, int ndim) +{ + // PyObject *tmp_result = PyList_New(0); + if (ndim == 1) + { + if (!result) + return nullptr; + + Py_ssize_t fillsz = view->len / view->itemsize; + PyObject *pb_call = PyObject_GetAttrString(self, "push_back"); + + for (Py_ssize_t i = 0; i < fillsz; i++) + { + int val = Get_IndexValue(view, i); + PyObject *item = PyLong_FromLong(val); + + if (!item) + { + Py_DECREF(result); + Py_XDECREF(pb_call); + return nullptr; + } + + PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); + Py_DECREF(item); + + if (!pbres) + { + Py_DECREF(result); + Py_XDECREF(pb_call); + return nullptr; + } + + Py_DECREF(pbres); + } + + Py_XDECREF(pb_call); + return result; + } + + if (!result) + return nullptr; + + Py_ssize_t *shape = (Py_ssize_t *)view->shape; + Py_ssize_t *strides = (Py_ssize_t *)view->strides; + Py_ssize_t *subshape = shape + 1; + Py_ssize_t *substrides = strides + 1; + + for (Py_ssize_t i = 0; i < view->ndim; i++) + { + Py_buffer subview; + subview.buf = (void *)((char *)view->buf + i * strides[0]); + subview.obj = NULL; + subview.len = subshape[0] * substrides[0]; + subview.readonly = view->readonly; + subview.itemsize = view->itemsize; + subview.format = view->format; + subview.ndim = ndim - 1; + subview.shape = subshape; + subview.strides = substrides; + subview.suboffsets = view->suboffsets; + subview.internal = view->internal; + + printf("ndim: %d\n", view->ndim); + printf("shape: %ld, %ld\n", shape[0], shape[1]); + printf("strides: %ld, %ld\n", strides[0], strides[1]); + printf("itemsize: %ld\n", view->itemsize); + + PyObject *subresult = recursive_vector_init(self, &subview, result, ndim - 1); + if (!subresult) + { + Py_DECREF(result); + return nullptr; + } + PyObject *pb_call = PyObject_GetAttrString(self, "push_back"); + + + PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, subresult, nullptr); + + if (!pbres) + { + Py_DECREF(result); + Py_XDECREF(pb_call); + return nullptr; + } + + Py_DECREF(pbres); + Py_DECREF(pb_call); + + Py_DECREF(subresult); + } + + return result; +} PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) { @@ -534,112 +618,19 @@ PyObject* VectorInit(PyObject* self, PyObject* args, PyObject* /* kwds */) // check if memoryview is valid if (view->buf == NULL) return nullptr; + + PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - // logic to handle 1-dim buffer (i.e. numpy array) - if (view->ndim == 1) - { - // Create a new vector object - PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - if (!result) return nullptr; - - // number of elements in the buffer - Py_ssize_t fillsz = view->len / view->itemsize; - - // push_back attribute - PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); - - for (Py_ssize_t i = 0; i < fillsz; i++) - { - // accessing item for buffer - double val = Get_IndexValue(view, i); - PyObject *item = PyLong_FromDouble(val); - - if (!item) - { - Py_DECREF(result); - return nullptr; - } - // element push back - PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item, nullptr); + result = recursive_vector_init(self, view, result, view->ndim); - if (!pbres) - { - Py_DECREF(result); - break; - return nullptr; - } - - Py_DECREF(item); - } - - // dereference the memoryview buffer - Py_DECREF(memoryview); - PyBuffer_Release(view); - Py_DECREF(fi); - - return result; - } - // logic to handle n-dim buffer (i.e. numpy array of arrays) - else - { - PyObject *result = PyObject_CallMethodNoArgs(self, PyStrings::gRealInit); - PyObject *pb_call = PyObject_GetAttrString(self, (char *)"push_back"); - - if (!result) - { - Py_DECREF(memoryview); - PyBuffer_Release(view); - Py_DECREF(fi); - return nullptr; - } - - for (Py_ssize_t i = 0; i < view->ndim; i++) { - // creating a tmp list for each dimension to pass to push_back - PyObject *item_vec = PyList_New(view->shape[1]); - - if (!item_vec) { - Py_DECREF(result); - Py_DECREF(memoryview); - PyBuffer_Release(view); - Py_DECREF(fi); - return nullptr; - } - - // accessing item for buffer - for (Py_ssize_t j = 0; j < view->shape[1]; j++) { - - int idx = i * view->shape[1] + j; - double val = Get_IndexValue(view, idx); - PyObject *item = PyLong_FromDouble(val); - - if (!item) { - Py_DECREF(result); - Py_DECREF(item_vec); - return nullptr; - } - PyList_SetItem(item_vec, j, item); - Py_DECREF(item); - } - - // element push back - PyObject *pbres = PyObject_CallFunctionObjArgs(pb_call, item_vec, nullptr); - if (!pbres) { - Py_DECREF(result); - Py_DECREF(item_vec); - break; - return nullptr; - } - } - // dereference the memoryview buffer - Py_DECREF(memoryview); - PyBuffer_Release(view); - Py_DECREF(fi); - - return result; - } + // dereference the memoryview buffer + PyBuffer_Release(view); + Py_DECREF(memoryview); + Py_DECREF(fi); + + return result; + } - - // The given argument wasn't iterable or a numpy array: simply forward to regular constructor PyObject *realInit = PyObject_GetAttr(self, PyStrings::gRealInit); From 8101d938f9c44af488578f0356f3337d66ca574a Mon Sep 17 00:00:00 2001 From: Khushiyant Date: Tue, 20 Aug 2024 21:15:13 +0530 Subject: [PATCH 19/19] chore: remove print statements --- src/Pythonize.cxx | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Pythonize.cxx b/src/Pythonize.cxx index 8e6c22e9..9a06e980 100644 --- a/src/Pythonize.cxx +++ b/src/Pythonize.cxx @@ -540,11 +540,6 @@ PyObject *recursive_vector_init(PyObject *self, Py_buffer *view, PyObject *resul subview.suboffsets = view->suboffsets; subview.internal = view->internal; - printf("ndim: %d\n", view->ndim); - printf("shape: %ld, %ld\n", shape[0], shape[1]); - printf("strides: %ld, %ld\n", strides[0], strides[1]); - printf("itemsize: %ld\n", view->itemsize); - PyObject *subresult = recursive_vector_init(self, &subview, result, ndim - 1); if (!subresult) {