Import array is moved to the Cython thread, and memory is put in a capsule, also on Linux. I don't know why, but it seemed to fix the memory leak

lasp/CMakeLists.txt

@@ -6,7 +6,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 set(CYTHON_EXECUTABLE "cython3")
 include(UseCython)
-find_package(Numpy REQUIRED )
+include(FindNumpy)
 
 include_directories(
     ${PYTHON_NUMPY_INCLUDE_DIR}

lasp/c/lasp_pyarray.h

@@ -9,16 +9,18 @@
 #pragma once
 #ifndef LASP_PYARRAY_H
 #define LASP_PYARRAY_H
-#include <numpy/ndarrayobject.h>
 #include "lasp_types.h"
+#include <assert.h>
+#include <numpy/ndarrayobject.h>
+
 #if LASP_DOUBLE_PRECISION == 1
 #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT64
 #define LASP_NUMPY_COMPLEX_TYPE NPY_COMPLEX128
 #else
 #define LASP_NUMPY_FLOAT_TYPE NPY_FLOAT32
 #endif
+#include <malloc.h>
 
-#ifdef MS_WIN64
 /**
  * Function passed to Python to use for cleanup of
  * foreignly obtained data.
@@ -29,54 +31,48 @@ static inline void capsule_cleanup(PyObject *capsule) {
   free(memory);
 }
 
-#endif
+/**
-
-
-/** 
  * Create a numpy array from a raw data pointer.
  *
  * @param data pointer to data, assumes d* for data
- * @param transfer_ownership If set to true, Numpy array will be responsible
+ * @param transfer_ownership If set to true, the created Numpy array will be
- * for freeing the data.
+ * responsible for freeing the data.
  *
  * @return Numpy array
  */
-static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
+static inline PyObject *data_to_ndarray(void *data, int n_rows, int n_cols,
     int typenum, bool transfer_ownership,
     bool F_contiguous) {
 
   /* fprintf(stderr, "Enter data_to_ndarray\n"); */
   assert(data);
-  import_array();
+
   PyArray_Descr *descr = PyArray_DescrFromType(typenum);
-  if(!descr) return NULL;
+  if (!descr)
+    return NULL;
 
   npy_intp dims[2] = {n_rows, n_cols};
   npy_intp strides[2];
 
   int flags = 0;
-  if(F_contiguous){
+  if (F_contiguous) {
-    flags |= NPY_ARRAY_FARRAY; 
+    flags |= NPY_ARRAY_FARRAY;
     strides[0] = descr->elsize;
-    strides[1] = descr->elsize*n_rows;
+    strides[1] = descr->elsize * n_rows;
   } else {
-    strides[0] = descr->elsize*n_rows;
+    strides[0] = descr->elsize * n_rows;
     strides[1] = descr->elsize;
   }
 
-  /* assert(n_rows > 0); */
-  /* assert(n_cols > 0); */
-
   PyArrayObject *arr =
-    (PyArrayObject *)PyArray_NewFromDescr(
+    (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type,
-        &PyArray_Type,
+        descr,   // Description
-        descr, // Description
+        2,       // nd
-        2, // nd
+        dims,    // dimensions
-        dims, // dimensions
+        strides, // strides
-        strides,  // strides
+        data,    // Data pointer
-        data, // Data pointer
+        flags,   // Flags
-        flags, // Flags
+        NULL     // obj
-        NULL // obj
         );
 
   if (!arr) {
@@ -84,8 +80,7 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
     return NULL;
   }
 
-  if (transfer_ownership == true) {
+  if (transfer_ownership) {
-    #ifdef MS_WIN64
     // The default destructor of Python cannot free the data, as it is allocated
     // with malloc. Therefore, with this code, we tell Numpy/Python to use
     // the capsule_cleanup constructor. See:
@@ -94,19 +89,16 @@ static inline PyObject *data_to_ndarray(d *data, int n_rows, int n_cols,
     // Windows. We do it anyway, see if we find any problems on the way.
     PyObject *capsule = PyCapsule_New(data, LASP_CAPSULE_NAME, capsule_cleanup);
     int res = PyArray_SetBaseObject(arr, capsule);
-    if(res != 0) {
+    if (res != 0) {
       fprintf(stderr, "Failed to set base object of array!");
       return NULL;
     }
-    #endif
+
-    /* fprintf(stderr, "============Ownership transfer================\n"); */
     PyArray_ENABLEFLAGS(arr, NPY_OWNDATA);
   }
-  /* fprintf(stderr, "Exit data_to_ndarray\n"); */
 
-  return (PyObject *) arr;
+  return (PyObject *)arr;
 }
 
 #undef LASP_CAPSULE_NAME
 #endif // LASP_PYARRAY_H
-//////////////////////////////////////////////////////////////////////

lasp/c/lasp_python.h

@@ -1,6 +1,6 @@
 // ascee_python.h
 //
-// Author: J.A. de Jong - ASCEE
+// Author: J.A. de Jong - ASCEE - Redu-Sone
 //
 // Description:
 // Some routines to generate numpy arrays from matrices and vectors.
@@ -8,16 +8,28 @@
 #pragma once
 #ifndef LASP_PYTHON_H
 #define LASP_PYTHON_H
+
+#ifdef __cplusplus
+#error "Cannot compile this file with C++"
+#endif
+#include "lasp_types.h"
+#include "lasp_mat.h"
 #include "lasp_pyarray.h"
 
-/** 
+/**
- * Create a numpy array from an existing dmat.
+ * @brief Create a numpy array from a buffer of floating point data
  *
- * @param mat dmat structure containing array data and metadata.
+ * @param data Pointer 
- * @param transfer_ownership If set to true, Numpy array will be responsible
+ * @param n_rows Number of columns in the data
- * for freeing the data.
+ * @param n_cols Number of rows in the data
+ * @param transfer_ownership If set to true, the created Numpy array will
+ * obtain ownership of the data and calls free() on destruction.
+ * @param F_contiguous It set to true, the data is assumed to be column-major
+ * ordered in memory (which means we can find element d[r,c] by d[n_cols*r+c],
+ * if set to false. Data is assumed to be row-major ordered (which means we
+ * find element d[r,c] by d[n_rows*c+r]
  *
- * @return Numpy array
+ * @return 
  */
 static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
     dbgassert(mat,NULLPTRDEREF);
@@ -48,6 +60,5 @@ static inline PyObject* dmat_to_ndarray(dmat* mat,bool transfer_ownership) {
     return arr;
 }
 
-
 #endif // LASP_PYTHON_H
 //////////////////////////////////////////////////////////////////////

lasp/cmake/FindNumpy.cmake

@@ -3,7 +3,7 @@
 # PYTHON_NUMPY_FOUND
 # will be set by this script
 
-cmake_minimum_required(VERSION 2.6)
+cmake_minimum_required(VERSION 3.0)
 
 if(NOT PYTHON_EXECUTABLE)
   if(NumPy_FIND_QUIETLY)

lasp/device/lasp_common_decls.pxd

@@ -35,7 +35,8 @@ cdef extern from "lasp_pyarray.h":
                               int n_rows,int n_cols,
                               int typenum,
                               bool transfer_ownership,
-                              bool F_contiguous)
+                              bool F_contiguous) with gil
+
 
 ctypedef size_t us
 ctypedef vector[bool] boolvec

lasp/device/lasp_cpprtaudio.cpp

@@ -1,9 +1,9 @@
 #include "lasp_cpprtaudio.h"
 #include <RtAudio.h>
 #include <atomic>
-#include <thread>
-#include <cstring>
 #include <cassert>
+#include <cstring>
+#include <thread>
 
 #if MS_WIN64
 typedef uint8_t u_int8_t;

lasp/device/lasp_cppuldaq.cpp

@@ -447,7 +447,7 @@ void threadfcn(DT9837A *td) {
       }
       inTotalCount = inxstat.currentScanCount;
 
-      if (inxstat.currentIndex < buffer_mid_idx_in) {
+      if ((us) inxstat.currentIndex < buffer_mid_idx_in) {
         topinenqueued = false;
         if (!botinenqueued) {
           /* cerr << "Copying in buffer bot" << endl; */

lasp/device/lasp_daq.pyx

@@ -2,6 +2,7 @@ cimport cython
 from ..lasp_common import AvType
 from .lasp_deviceinfo cimport DeviceInfo
 from .lasp_daqconfig cimport DaqConfiguration
+from numpy cimport import_array
 
 from cpython.ref cimport PyObject,Py_INCREF, Py_DECREF
 import numpy as np
@@ -88,6 +89,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
                                          sd.nFramesPerBlock))
 
     with gil:
+        import_array()
         npy_format = cnp.NPY_FLOAT64
         callback = <object> sd.pyCallback
         # print(f'Number of input channels: {ninchannels}')
@@ -116,10 +118,7 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
                            # Numpy container
                     True) # F-contiguous
                     try:
-                        rval = callback(None,
+                        rval = callback(None, npy_output, nFramesPerBlock)
-                        npy_output,
-                        nFramesPerBlock,
-                        )
 
                     except Exception as e:
                         logging.error('exception in Cython callback for audio output: ', str(e))
@@ -131,21 +130,19 @@ cdef void audioCallbackPythonThreadFunction(void* voidsd) nogil:
                 # Waiting indefinitely on the queue...
                 inbuffer = <double*> sd.inQueue.dequeue()
                 if inbuffer == NULL:
-                    logging.debug('Stopping thread...\n')
+                    logging.debug('Received empty buffer on input, stopping thread...\n')
                     return
 
+                npy_input = <object> data_to_ndarray(
+                                inbuffer,
+                                nFramesPerBlock,
+                                ninchannels,
+                                sd.npy_format,
+                                True, # Do transfer ownership
+                                True) # F-contiguous is True: data is Fortran-cont.
+
                 try:
-                    npy_input = <object> data_to_ndarray(
+                    rval = callback(npy_input, None, nFramesPerBlock)
-                    inbuffer,
-                    nFramesPerBlock,
-                    ninchannels,
-                    sd.npy_format,
-                    True, # Do transfer ownership
-                    True) # F-contiguous is True: data is Fortran-cont.
-                    rval = callback(npy_input,
-                    None,
-                    nFramesPerBlock,
-                    )
 
                 except Exception as e:
                     logging.error('exception in cython callback for audio input: ', str(e))