From 13b02437217382a29d549b938daa7b9f5ede7fb8 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Wed, 2 Feb 2022 12:38:29 +0100 Subject: [PATCH] Import array is moved to the Cython thread, and memory is put in a capsule, also on Linux. I don't know why, but it seemed to fix the memory leak --- lasp/CMakeLists.txt | 2 +- lasp/c/lasp_pyarray.h | 62 ++++++++++++++----------------- lasp/c/lasp_python.h | 27 ++++++++++---- lasp/cmake/FindNumpy.cmake | 2 +- lasp/device/lasp_common_decls.pxd | 3 +- lasp/device/lasp_cpprtaudio.cpp | 4 +- lasp/device/lasp_cppuldaq.cpp | 2 +- lasp/device/lasp_daq.pyx | 29 +++++++-------- 8 files changed, 66 insertions(+), 65 deletions(-) diff --git a/lasp/CMakeLists.txt b/lasp/CMakeLists.txt index 0f48751..23c6865 100644 --- a/lasp/CMakeLists.txt +++ b/lasp/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CYTHON_EXECUTABLE "cython3") include(UseCython) -find_package(Numpy REQUIRED ) +include(FindNumpy) include_directories( ${PYTHON_NUMPY_INCLUDE_DIR} diff --git a/lasp/c/lasp_pyarray.h b/lasp/c/lasp_pyarray.h index 3ddeba4..90c9205 100644 --- a/lasp/c/lasp_pyarray.h +++ b/lasp/c/lasp_pyarray.h @@ -9,16 +9,18 @@ #pragma once #ifndef LASP_PYARRAY_H #define LASP_PYARRAY_H -#include #include "lasp_types.h" +#include +#include + #if LASP_DOUBLE_PRECISION == 1 #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT64 #define LASP_NUMPY_COMPLEX_TYPE NPY_COMPLEX128 #else #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT32 #endif +#include -#ifdef MS_WIN64 /** * Function passed to Python to use for cleanup of * foreignly obtained data. @@ -29,54 +31,48 @@ static inline void capsule_cleanup(PyObject *capsule) { free(memory); } -#endif - - -/** +/** * Create a numpy array from a raw data pointer. * * @param data pointer to data, assumes d* for data - * @param transfer_ownership If set to true, Numpy array will be responsible - * for freeing the data. + * @param transfer_ownership If set to true, the created Numpy array will be + * responsible for freeing the data. * * @return Numpy array */ -static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols, +static inline PyObject *data_to_ndarray(void *data, int n_rows, int n_cols, int typenum, bool transfer_ownership, bool F_contiguous) { /* fprintf(stderr, "Enter data_to_ndarray\n"); */ assert(data); - import_array(); + PyArray_Descr *descr = PyArray_DescrFromType(typenum); - if(!descr) return NULL; + if (!descr) + return NULL; npy_intp dims[2] = {n_rows, n_cols}; npy_intp strides[2]; int flags = 0; - if(F_contiguous){ - flags |= NPY_ARRAY_FARRAY; + if (F_contiguous) { + flags |= NPY_ARRAY_FARRAY; strides[0] = descr->elsize; - strides[1] = descr->elsize*n_rows; + strides[1] = descr->elsize * n_rows; } else { - strides[0] = descr->elsize*n_rows; + strides[0] = descr->elsize * n_rows; strides[1] = descr->elsize; } - /* assert(n_rows > 0); */ - /* assert(n_cols > 0); */ - PyArrayObject *arr = - (PyArrayObject *)PyArray_NewFromDescr( - &PyArray_Type, - descr, // Description - 2, // nd - dims, // dimensions - strides, // strides - data, // Data pointer - flags, // Flags - NULL // obj + (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, + descr, // Description + 2, // nd + dims, // dimensions + strides, // strides + data, // Data pointer + flags, // Flags + NULL // obj ); if (!arr) { @@ -84,8 +80,7 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols, return NULL; } - if (transfer_ownership == true) { - #ifdef MS_WIN64 + if (transfer_ownership) { // The default destructor of Python cannot free the data, as it is allocated // with malloc. Therefore, with this code, we tell Numpy/Python to use // the capsule_cleanup constructor. See: @@ -94,19 +89,16 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols, // Windows. We do it anyway, see if we find any problems on the way. PyObject *capsule = PyCapsule_New(data, LASP_CAPSULE_NAME, capsule_cleanup); int res = PyArray_SetBaseObject(arr, capsule); - if(res != 0) { + if (res != 0) { fprintf(stderr, "Failed to set base object of array!"); return NULL; } - #endif - /* fprintf(stderr, "============Ownership transfer================\n"); */ + PyArray_ENABLEFLAGS(arr, NPY_OWNDATA); } - /* fprintf(stderr, "Exit data_to_ndarray\n"); */ - return (PyObject *) arr; + return (PyObject *)arr; } #undef LASP_CAPSULE_NAME #endif // LASP_PYARRAY_H -////////////////////////////////////////////////////////////////////// diff --git a/lasp/c/lasp_python.h b/lasp/c/lasp_python.h index 409b082..bdd1871 100644 --- a/lasp/c/lasp_python.h +++ b/lasp/c/lasp_python.h @@ -1,6 +1,6 @@ // ascee_python.h // -// Author: J.A. de Jong - ASCEE +// Author: J.A. de Jong - ASCEE - Redu-Sone // // Description: // Some routines to generate numpy arrays from matrices and vectors. @@ -8,16 +8,28 @@ #pragma once #ifndef LASP_PYTHON_H #define LASP_PYTHON_H + +#ifdef __cplusplus +#error "Cannot compile this file with C++" +#endif +#include "lasp_types.h" +#include "lasp_mat.h" #include "lasp_pyarray.h" -/** - * Create a numpy array from an existing dmat. +/** + * @brief Create a numpy array from a buffer of floating point data * - * @param mat dmat structure containing array data and metadata. - * @param transfer_ownership If set to true, Numpy array will be responsible - * for freeing the data. + * @param data Pointer + * @param n_rows Number of columns in the data + * @param n_cols Number of rows in the data + * @param transfer_ownership If set to true, the created Numpy array will + * obtain ownership of the data and calls free() on destruction. + * @param F_contiguous It set to true, the data is assumed to be column-major + * ordered in memory (which means we can find element d[r,c] by d[n_cols*r+c], + * if set to false. Data is assumed to be row-major ordered (which means we + * find element d[r,c] by d[n_rows*c+r] * - * @return Numpy array + * @return */ static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) { dbgassert(mat,NULLPTRDEREF); @@ -48,6 +60,5 @@ static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) { return arr; } - #endif // LASP_PYTHON_H ////////////////////////////////////////////////////////////////////// diff --git a/lasp/cmake/FindNumpy.cmake b/lasp/cmake/FindNumpy.cmake index a95d6ac..c673ca1 100644 --- a/lasp/cmake/FindNumpy.cmake +++ b/lasp/cmake/FindNumpy.cmake @@ -3,7 +3,7 @@ # PYTHON_NUMPY_FOUND # will be set by this script -cmake_minimum_required(VERSION 2.6) +cmake_minimum_required(VERSION 3.0) if(NOT PYTHON_EXECUTABLE) if(NumPy_FIND_QUIETLY) diff --git a/lasp/device/lasp_common_decls.pxd b/lasp/device/lasp_common_decls.pxd index 5fec933..53b197e 100644 --- a/lasp/device/lasp_common_decls.pxd +++ b/lasp/device/lasp_common_decls.pxd @@ -35,7 +35,8 @@ cdef extern from "lasp_pyarray.h": int n_rows,int n_cols, int typenum, bool transfer_ownership, - bool F_contiguous) + bool F_contiguous) with gil + ctypedef size_t us ctypedef vector[bool] boolvec diff --git a/lasp/device/lasp_cpprtaudio.cpp b/lasp/device/lasp_cpprtaudio.cpp index a12ca76..d630008 100644 --- a/lasp/device/lasp_cpprtaudio.cpp +++ b/lasp/device/lasp_cpprtaudio.cpp @@ -1,9 +1,9 @@ #include "lasp_cpprtaudio.h" #include #include -#include -#include #include +#include +#include #if MS_WIN64 typedef uint8_t u_int8_t; diff --git a/lasp/device/lasp_cppuldaq.cpp b/lasp/device/lasp_cppuldaq.cpp index b67d671..b68a861 100644 --- a/lasp/device/lasp_cppuldaq.cpp +++ b/lasp/device/lasp_cppuldaq.cpp @@ -447,7 +447,7 @@ void threadfcn(DT9837A *td) { } inTotalCount = inxstat.currentScanCount; - if (inxstat.currentIndex < buffer_mid_idx_in) { + if ((us) inxstat.currentIndex < buffer_mid_idx_in) { topinenqueued = false; if (!botinenqueued) { /* cerr << "Copying in buffer bot" << endl; */ diff --git a/lasp/device/lasp_daq.pyx b/lasp/device/lasp_daq.pyx index 6557583..f1d094d 100644 --- a/lasp/device/lasp_daq.pyx +++ b/lasp/device/lasp_daq.pyx @@ -2,6 +2,7 @@ cimport cython from ..lasp_common import AvType from .lasp_deviceinfo cimport DeviceInfo from .lasp_daqconfig cimport DaqConfiguration +from numpy cimport import_array from cpython.ref cimport PyObject,Py_INCREF, Py_DECREF import numpy as np @@ -88,6 +89,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil: sd.nFramesPerBlock)) with gil: + import_array() npy_format = cnp.NPY_FLOAT64 callback = sd.pyCallback # print(f'Number of input channels: {ninchannels}') @@ -116,10 +118,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil: # Numpy container True) # F-contiguous try: - rval = callback(None, - npy_output, - nFramesPerBlock, - ) + rval = callback(None, npy_output, nFramesPerBlock) except Exception as e: logging.error('exception in Cython callback for audio output: ', str(e)) @@ -131,21 +130,19 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil: # Waiting indefinitely on the queue... inbuffer = sd.inQueue.dequeue() if inbuffer == NULL: - logging.debug('Stopping thread...\n') + logging.debug('Received empty buffer on input, stopping thread...\n') return + npy_input = data_to_ndarray( + inbuffer, + nFramesPerBlock, + ninchannels, + sd.npy_format, + True, # Do transfer ownership + True) # F-contiguous is True: data is Fortran-cont. + try: - npy_input = data_to_ndarray( - inbuffer, - nFramesPerBlock, - ninchannels, - sd.npy_format, - True, # Do transfer ownership - True) # F-contiguous is True: data is Fortran-cont. - rval = callback(npy_input, - None, - nFramesPerBlock, - ) + rval = callback(npy_input, None, nFramesPerBlock) except Exception as e: logging.error('exception in cython callback for audio input: ', str(e))