Import array is moved to the Cython thread, and memory is put in a capsule, also on Linux. I don't know why, but it seemed to fix the memory leak

This commit is contained in:
Anne de Jong 2022-02-02 12:38:29 +01:00
parent e0f74121fe
commit 13b0243721
8 changed files with 66 additions and 65 deletions

View File

@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(CYTHON_EXECUTABLE "cython3") set(CYTHON_EXECUTABLE "cython3")
include(UseCython) include(UseCython)
find_package(Numpy REQUIRED ) include(FindNumpy)
include_directories( include_directories(
${PYTHON_NUMPY_INCLUDE_DIR} ${PYTHON_NUMPY_INCLUDE_DIR}

View File

@ -9,16 +9,18 @@
#pragma once #pragma once
#ifndef LASP_PYARRAY_H #ifndef LASP_PYARRAY_H
#define LASP_PYARRAY_H #define LASP_PYARRAY_H
#include <numpy/ndarrayobject.h>
#include "lasp_types.h" #include "lasp_types.h"
#include <assert.h>
#include <numpy/ndarrayobject.h>
#if LASP_DOUBLE_PRECISION == 1 #if LASP_DOUBLE_PRECISION == 1
#define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT64 #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT64
#define LASP_NUMPY_COMPLEX_TYPE NPY_COMPLEX128 #define LASP_NUMPY_COMPLEX_TYPE NPY_COMPLEX128
#else #else
#define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT32 #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT32
#endif #endif
#include <malloc.h>
#ifdef MS_WIN64
/** /**
* Function passed to Python to use for cleanup of * Function passed to Python to use for cleanup of
* foreignly obtained data. * foreignly obtained data.
@ -29,47 +31,41 @@ static inline void capsule_cleanup(PyObject *capsule) {
free(memory); free(memory);
} }
#endif
/** /**
* Create a numpy array from a raw data pointer. * Create a numpy array from a raw data pointer.
* *
* @param data pointer to data, assumes d* for data * @param data pointer to data, assumes d* for data
* @param transfer_ownership If set to true, Numpy array will be responsible * @param transfer_ownership If set to true, the created Numpy array will be
* for freeing the data. * responsible for freeing the data.
* *
* @return Numpy array * @return Numpy array
*/ */
static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols, static inline PyObject *data_to_ndarray(void *data, int n_rows, int n_cols,
int typenum, bool transfer_ownership, int typenum, bool transfer_ownership,
bool F_contiguous) { bool F_contiguous) {
/* fprintf(stderr, "Enter data_to_ndarray\n"); */ /* fprintf(stderr, "Enter data_to_ndarray\n"); */
assert(data); assert(data);
import_array();
PyArray_Descr *descr = PyArray_DescrFromType(typenum); PyArray_Descr *descr = PyArray_DescrFromType(typenum);
if(!descr) return NULL; if (!descr)
return NULL;
npy_intp dims[2] = {n_rows, n_cols}; npy_intp dims[2] = {n_rows, n_cols};
npy_intp strides[2]; npy_intp strides[2];
int flags = 0; int flags = 0;
if(F_contiguous){ if (F_contiguous) {
flags |= NPY_ARRAY_FARRAY; flags |= NPY_ARRAY_FARRAY;
strides[0] = descr->elsize; strides[0] = descr->elsize;
strides[1] = descr->elsize*n_rows; strides[1] = descr->elsize * n_rows;
} else { } else {
strides[0] = descr->elsize*n_rows; strides[0] = descr->elsize * n_rows;
strides[1] = descr->elsize; strides[1] = descr->elsize;
} }
/* assert(n_rows > 0); */
/* assert(n_cols > 0); */
PyArrayObject *arr = PyArrayObject *arr =
(PyArrayObject *)PyArray_NewFromDescr( (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type,
&PyArray_Type,
descr, // Description descr, // Description
2, // nd 2, // nd
dims, // dimensions dims, // dimensions
@ -84,8 +80,7 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
return NULL; return NULL;
} }
if (transfer_ownership == true) { if (transfer_ownership) {
#ifdef MS_WIN64
// The default destructor of Python cannot free the data, as it is allocated // The default destructor of Python cannot free the data, as it is allocated
// with malloc. Therefore, with this code, we tell Numpy/Python to use // with malloc. Therefore, with this code, we tell Numpy/Python to use
// the capsule_cleanup constructor. See: // the capsule_cleanup constructor. See:
@ -94,19 +89,16 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
// Windows. We do it anyway, see if we find any problems on the way. // Windows. We do it anyway, see if we find any problems on the way.
PyObject *capsule = PyCapsule_New(data, LASP_CAPSULE_NAME, capsule_cleanup); PyObject *capsule = PyCapsule_New(data, LASP_CAPSULE_NAME, capsule_cleanup);
int res = PyArray_SetBaseObject(arr, capsule); int res = PyArray_SetBaseObject(arr, capsule);
if(res != 0) { if (res != 0) {
fprintf(stderr, "Failed to set base object of array!"); fprintf(stderr, "Failed to set base object of array!");
return NULL; return NULL;
} }
#endif
/* fprintf(stderr, "============Ownership transfer================\n"); */
PyArray_ENABLEFLAGS(arr, NPY_OWNDATA); PyArray_ENABLEFLAGS(arr, NPY_OWNDATA);
} }
/* fprintf(stderr, "Exit data_to_ndarray\n"); */
return (PyObject *) arr; return (PyObject *)arr;
} }
#undef LASP_CAPSULE_NAME #undef LASP_CAPSULE_NAME
#endif // LASP_PYARRAY_H #endif // LASP_PYARRAY_H
//////////////////////////////////////////////////////////////////////

View File

@ -1,6 +1,6 @@
// ascee_python.h // ascee_python.h
// //
// Author: J.A. de Jong - ASCEE // Author: J.A. de Jong - ASCEE - Redu-Sone
// //
// Description: // Description:
// Some routines to generate numpy arrays from matrices and vectors. // Some routines to generate numpy arrays from matrices and vectors.
@ -8,16 +8,28 @@
#pragma once #pragma once
#ifndef LASP_PYTHON_H #ifndef LASP_PYTHON_H
#define LASP_PYTHON_H #define LASP_PYTHON_H
#ifdef __cplusplus
#error "Cannot compile this file with C++"
#endif
#include "lasp_types.h"
#include "lasp_mat.h"
#include "lasp_pyarray.h" #include "lasp_pyarray.h"
/** /**
* Create a numpy array from an existing dmat. * @brief Create a numpy array from a buffer of floating point data
* *
* @param mat dmat structure containing array data and metadata. * @param data Pointer
* @param transfer_ownership If set to true, Numpy array will be responsible * @param n_rows Number of columns in the data
* for freeing the data. * @param n_cols Number of rows in the data
* @param transfer_ownership If set to true, the created Numpy array will
* obtain ownership of the data and calls free() on destruction.
* @param F_contiguous It set to true, the data is assumed to be column-major
* ordered in memory (which means we can find element d[r,c] by d[n_cols*r+c],
* if set to false. Data is assumed to be row-major ordered (which means we
* find element d[r,c] by d[n_rows*c+r]
* *
* @return Numpy array * @return
*/ */
static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) { static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
dbgassert(mat,NULLPTRDEREF); dbgassert(mat,NULLPTRDEREF);
@ -48,6 +60,5 @@ static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
return arr; return arr;
} }
#endif // LASP_PYTHON_H #endif // LASP_PYTHON_H
////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////

View File

@ -3,7 +3,7 @@
# PYTHON_NUMPY_FOUND # PYTHON_NUMPY_FOUND
# will be set by this script # will be set by this script
cmake_minimum_required(VERSION 2.6) cmake_minimum_required(VERSION 3.0)
if(NOT PYTHON_EXECUTABLE) if(NOT PYTHON_EXECUTABLE)
if(NumPy_FIND_QUIETLY) if(NumPy_FIND_QUIETLY)

View File

@ -35,7 +35,8 @@ cdef extern from "lasp_pyarray.h":
int n_rows,int n_cols, int n_rows,int n_cols,
int typenum, int typenum,
bool transfer_ownership, bool transfer_ownership,
bool F_contiguous) bool F_contiguous) with gil
ctypedef size_t us ctypedef size_t us
ctypedef vector[bool] boolvec ctypedef vector[bool] boolvec

View File

@ -1,9 +1,9 @@
#include "lasp_cpprtaudio.h" #include "lasp_cpprtaudio.h"
#include <RtAudio.h> #include <RtAudio.h>
#include <atomic> #include <atomic>
#include <thread>
#include <cstring>
#include <cassert> #include <cassert>
#include <cstring>
#include <thread>
#if MS_WIN64 #if MS_WIN64
typedef uint8_t u_int8_t; typedef uint8_t u_int8_t;

View File

@ -447,7 +447,7 @@ void threadfcn(DT9837A *td) {
} }
inTotalCount = inxstat.currentScanCount; inTotalCount = inxstat.currentScanCount;
if (inxstat.currentIndex < buffer_mid_idx_in) { if ((us) inxstat.currentIndex < buffer_mid_idx_in) {
topinenqueued = false; topinenqueued = false;
if (!botinenqueued) { if (!botinenqueued) {
/* cerr << "Copying in buffer bot" << endl; */ /* cerr << "Copying in buffer bot" << endl; */

View File

@ -2,6 +2,7 @@ cimport cython
from ..lasp_common import AvType from ..lasp_common import AvType
from .lasp_deviceinfo cimport DeviceInfo from .lasp_deviceinfo cimport DeviceInfo
from .lasp_daqconfig cimport DaqConfiguration from .lasp_daqconfig cimport DaqConfiguration
from numpy cimport import_array
from cpython.ref cimport PyObject,Py_INCREF, Py_DECREF from cpython.ref cimport PyObject,Py_INCREF, Py_DECREF
import numpy as np import numpy as np
@ -88,6 +89,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
sd.nFramesPerBlock)) sd.nFramesPerBlock))
with gil: with gil:
import_array()
npy_format = cnp.NPY_FLOAT64 npy_format = cnp.NPY_FLOAT64
callback = <object> sd.pyCallback callback = <object> sd.pyCallback
# print(f'Number of input channels: {ninchannels}') # print(f'Number of input channels: {ninchannels}')
@ -116,10 +118,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
# Numpy container # Numpy container
True) # F-contiguous True) # F-contiguous
try: try:
rval = callback(None, rval = callback(None, npy_output, nFramesPerBlock)
npy_output,
nFramesPerBlock,
)
except Exception as e: except Exception as e:
logging.error('exception in Cython callback for audio output: ', str(e)) logging.error('exception in Cython callback for audio output: ', str(e))
@ -131,10 +130,9 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
# Waiting indefinitely on the queue... # Waiting indefinitely on the queue...
inbuffer = <double*> sd.inQueue.dequeue() inbuffer = <double*> sd.inQueue.dequeue()
if inbuffer == NULL: if inbuffer == NULL:
logging.debug('Stopping thread...\n') logging.debug('Received empty buffer on input, stopping thread...\n')
return return
try:
npy_input = <object> data_to_ndarray( npy_input = <object> data_to_ndarray(
inbuffer, inbuffer,
nFramesPerBlock, nFramesPerBlock,
@ -142,10 +140,9 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
sd.npy_format, sd.npy_format,
True, # Do transfer ownership True, # Do transfer ownership
True) # F-contiguous is True: data is Fortran-cont. True) # F-contiguous is True: data is Fortran-cont.
rval = callback(npy_input,
None, try:
nFramesPerBlock, rval = callback(npy_input, None, nFramesPerBlock)
)
except Exception as e: except Exception as e:
logging.error('exception in cython callback for audio input: ', str(e)) logging.error('exception in cython callback for audio input: ', str(e))