ASCEE/lasp
ASCEE/lasp
5
1
Fork 0
Code Issues 4 Pull Requests Actions Releases 12 Activity

Import array is moved to the Cython thread, and memory is put in a capsule, also on Linux. I don't know why, but it seemed to fix the memory leak

Browse Source
This commit is contained in:
Anne de Jong 2022-02-02 12:38:29 +01:00
parent e0f74121fe
commit 13b0243721
8 changed files with 66 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lasp/CMakeLists.txt
View File

@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
set(CYTHON_EXECUTABLE "cython3")
include(UseCython)
find_package(Numpy REQUIRED )
include(FindNumpy)
include_directories(
${PYTHON_NUMPY_INCLUDE_DIR}

62
lasp/c/lasp_pyarray.h
View File

@ -9,16 +9,18 @@
#pragma once
#ifndef LASP_PYARRAY_H
#define LASP_PYARRAY_H
#include <numpy/ndarrayobject.h>
#include "lasp_types.h"
#include <assert.h>
#include <numpy/ndarrayobject.h>
#if LASP_DOUBLE_PRECISION == 1
#define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT64
#define LASP_NUMPY_COMPLEX_TYPE NPY_COMPLEX128
#else
#define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT32
#endif
#include <malloc.h>
#ifdef MS_WIN64
/**
* Function passed to Python to use for cleanup of
* foreignly obtained data.
@ -29,54 +31,48 @@ static inline void capsule_cleanup(PyObject *capsule) {
free(memory);
}
#endif
/**
/**
* Create a numpy array from a raw data pointer.
*
* @param data pointer to data, assumes d* for data
* @param transfer_ownership If set to true, Numpy array will be responsible
* for freeing the data.
* @param transfer_ownership If set to true, the created Numpy array will be
* responsible for freeing the data.
*
* @return Numpy array
*/
static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
static inline PyObject *data_to_ndarray(void *data, int n_rows, int n_cols,
int typenum, bool transfer_ownership,
bool F_contiguous) {
/* fprintf(stderr, "Enter data_to_ndarray\n"); */
assert(data);
import_array();
PyArray_Descr *descr = PyArray_DescrFromType(typenum);
if(!descr) return NULL;
if (!descr)
return NULL;
npy_intp dims[2] = {n_rows, n_cols};
npy_intp strides[2];
int flags = 0;
if(F_contiguous){
flags |= NPY_ARRAY_FARRAY;
if (F_contiguous) {
flags |= NPY_ARRAY_FARRAY;
strides[0] = descr->elsize;
strides[1] = descr->elsize*n_rows;
strides[1] = descr->elsize * n_rows;
} else {
strides[0] = descr->elsize*n_rows;
strides[0] = descr->elsize * n_rows;
strides[1] = descr->elsize;
}
/* assert(n_rows > 0); */
/* assert(n_cols > 0); */
PyArrayObject *arr =
(PyArrayObject *)PyArray_NewFromDescr(
&PyArray_Type,
descr, // Description
2, // nd
dims, // dimensions
strides, // strides
data, // Data pointer
flags, // Flags
NULL // obj
(PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type,
descr, // Description
2, // nd
dims, // dimensions
strides, // strides
data, // Data pointer
flags, // Flags
NULL // obj
);
if (!arr) {
@ -84,8 +80,7 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
return NULL;
}
if (transfer_ownership == true) {
#ifdef MS_WIN64
if (transfer_ownership) {
// The default destructor of Python cannot free the data, as it is allocated
// with malloc. Therefore, with this code, we tell Numpy/Python to use
// the capsule_cleanup constructor. See:
@ -94,19 +89,16 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
// Windows. We do it anyway, see if we find any problems on the way.
PyObject *capsule = PyCapsule_New(data, LASP_CAPSULE_NAME, capsule_cleanup);
int res = PyArray_SetBaseObject(arr, capsule);
if(res != 0) {
if (res != 0) {
fprintf(stderr, "Failed to set base object of array!");
return NULL;
}
#endif
/* fprintf(stderr, "============Ownership transfer================\n"); */
PyArray_ENABLEFLAGS(arr, NPY_OWNDATA);
}
/* fprintf(stderr, "Exit data_to_ndarray\n"); */
return (PyObject *) arr;
return (PyObject *)arr;
}
#undef LASP_CAPSULE_NAME
#endif // LASP_PYARRAY_H
//////////////////////////////////////////////////////////////////////

27
lasp/c/lasp_python.h
View File

@ -1,6 +1,6 @@
// ascee_python.h
//
// Author: J.A. de Jong - ASCEE
// Author: J.A. de Jong - ASCEE - Redu-Sone
//
// Description:
// Some routines to generate numpy arrays from matrices and vectors.
@ -8,16 +8,28 @@
#pragma once
#ifndef LASP_PYTHON_H
#define LASP_PYTHON_H
#ifdef __cplusplus
#error "Cannot compile this file with C++"
#endif
#include "lasp_types.h"
#include "lasp_mat.h"
#include "lasp_pyarray.h"
/**
* Create a numpy array from an existing dmat.
/**
* @brief Create a numpy array from a buffer of floating point data
*
* @param mat dmat structure containing array data and metadata.
* @param transfer_ownership If set to true, Numpy array will be responsible
* for freeing the data.
* @param data Pointer
* @param n_rows Number of columns in the data
* @param n_cols Number of rows in the data
* @param transfer_ownership If set to true, the created Numpy array will
* obtain ownership of the data and calls free() on destruction.
* @param F_contiguous It set to true, the data is assumed to be column-major
* ordered in memory (which means we can find element d[r,c] by d[n_cols*r+c],
* if set to false. Data is assumed to be row-major ordered (which means we
* find element d[r,c] by d[n_rows*c+r]
*
* @return Numpy array
* @return
*/
static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
dbgassert(mat,NULLPTRDEREF);
@ -48,6 +60,5 @@ static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
return arr;
}
#endif // LASP_PYTHON_H
//////////////////////////////////////////////////////////////////////

2
lasp/cmake/FindNumpy.cmake
View File

@ -3,7 +3,7 @@
# PYTHON_NUMPY_FOUND
# will be set by this script
cmake_minimum_required(VERSION 2.6)
cmake_minimum_required(VERSION 3.0)
if(NOT PYTHON_EXECUTABLE)
if(NumPy_FIND_QUIETLY)

3
lasp/device/lasp_common_decls.pxd
View File

@ -35,7 +35,8 @@ cdef extern from "lasp_pyarray.h":
int n_rows,int n_cols,
int typenum,
bool transfer_ownership,
bool F_contiguous)
bool F_contiguous) with gil
ctypedef size_t us
ctypedef vector[bool] boolvec

4
lasp/device/lasp_cpprtaudio.cpp
View File

@ -1,9 +1,9 @@
#include "lasp_cpprtaudio.h"
#include <RtAudio.h>
#include <atomic>
#include <thread>
#include <cstring>
#include <cassert>
#include <cstring>
#include <thread>
#if MS_WIN64
typedef uint8_t u_int8_t;

2
lasp/device/lasp_cppuldaq.cpp
View File

@ -447,7 +447,7 @@ void threadfcn(DT9837A *td) {
}
inTotalCount = inxstat.currentScanCount;
if (inxstat.currentIndex < buffer_mid_idx_in) {
if ((us) inxstat.currentIndex < buffer_mid_idx_in) {
topinenqueued = false;
if (!botinenqueued) {
/* cerr << "Copying in buffer bot" << endl; */

29
lasp/device/lasp_daq.pyx
View File

@ -2,6 +2,7 @@ cimport cython
from ..lasp_common import AvType
from .lasp_deviceinfo cimport DeviceInfo
from .lasp_daqconfig cimport DaqConfiguration
from numpy cimport import_array
from cpython.ref cimport PyObject,Py_INCREF, Py_DECREF
import numpy as np
@ -88,6 +89,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
sd.nFramesPerBlock))
with gil:
import_array()
npy_format = cnp.NPY_FLOAT64
callback = <object> sd.pyCallback
# print(f'Number of input channels: {ninchannels}')
@ -116,10 +118,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
# Numpy container
True) # F-contiguous
try:
rval = callback(None,
npy_output,
nFramesPerBlock,
)
rval = callback(None, npy_output, nFramesPerBlock)
except Exception as e:
logging.error('exception in Cython callback for audio output: ', str(e))
@ -131,21 +130,19 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
# Waiting indefinitely on the queue...
inbuffer = <double*> sd.inQueue.dequeue()
if inbuffer == NULL:
logging.debug('Stopping thread...\n')
logging.debug('Received empty buffer on input, stopping thread...\n')
return
npy_input = <object> data_to_ndarray(
inbuffer,
nFramesPerBlock,
ninchannels,
sd.npy_format,
True, # Do transfer ownership
True) # F-contiguous is True: data is Fortran-cont.
try:
npy_input = <object> data_to_ndarray(
inbuffer,
nFramesPerBlock,
ninchannels,
sd.npy_format,
True, # Do transfer ownership
True) # F-contiguous is True: data is Fortran-cont.
rval = callback(npy_input,
None,
nFramesPerBlock,
)
rval = callback(npy_input, None, nFramesPerBlock)
except Exception as e:
logging.error('exception in cython callback for audio input: ', str(e))