Some more bugfixes: weak references stored in indatahandler, to avoid calling destructor from wrong thread. Removed some unneccessary include statements on the way

cpp_src/dsp/lasp_avpowerspectra.cpp

@@ -2,12 +2,9 @@
 #include "lasp_avpowerspectra.h"
 #include "debugtrace.hpp"
 #include "lasp_mathtypes.h"
-#include <cmath>
-#include <optional>
+#include <stdexcept>
 
 using rte = std::runtime_error;
-using std::cerr;
-using std::endl;
 
 PowerSpectra::PowerSpectra(const us nfft, const Window::WindowType w)
     : PowerSpectra(Window::create(w, nfft)) {}

cpp_src/dsp/lasp_avpowerspectra.h

@@ -3,8 +3,6 @@
 #include "lasp_mathtypes.h"
 #include "lasp_timebuffer.h"
 #include "lasp_window.h"
-#include <memory>
-#include <optional>
 
 /** \defgroup dsp Digital Signal Processing utilities
  * These are classes and functions used for processing raw signal data, to

cpp_src/dsp/lasp_biquadbank.cpp

@@ -2,7 +2,6 @@
 #include "lasp_biquadbank.h"
 #include "debugtrace.hpp"
 #include "lasp_thread.h"
-#include <cassert>
 #include <vector>
 
 using std::cerr;

cpp_src/dsp/lasp_rtsignalviewer.h

@@ -4,12 +4,9 @@
 //
 // Description: Real Time Signal Viewer.
 #pragma once
-#include "lasp_avpowerspectra.h"
-#include "lasp_filter.h"
 #include "lasp_mathtypes.h"
 #include "lasp_threadedindatahandler.h"
 #include "lasp_timebuffer.h"
-#include <memory>
 #include <mutex>
 
 /**

cpp_src/pybind11/lasp_pyindatahandler.cpp

@@ -1,4 +1,10 @@
-/* #define DEBUGTRACE_ENABLED */
+// #define DEBUGTRACE_ENABLED
+#include <pybind11/pybind11.h>
+#include <pybind11/pytypes.h>
+
+#include <armadillo>
+#include <atomic>
+
 #include "arma_npy.h"
 #include "debugtrace.hpp"
 #include "lasp_clip.h"
@@ -9,10 +15,6 @@
 #include "lasp_rtsignalviewer.h"
 #include "lasp_streammgr.h"
 #include "lasp_threadedindatahandler.h"
-#include <armadillo>
-#include <atomic>
-#include <chrono>
-#include <pybind11/pybind11.h>
 
 using namespace std::literals::chrono_literals;
 using std::cerr;
@@ -104,8 +106,8 @@ class PyIndataHandler : public ThreadedInDataHandler<PyIndataHandler> {
   /**
    * @brief The callback functions that is called.
    */
-  std::unique_ptr<py::function> cb, reset_callback;
-  bool _done{false};
+  py::weakref _cb, _reset_callback;
+  std::atomic<bool> _done{false};
 
  public:
   /**
@@ -118,9 +120,7 @@ class PyIndataHandler : public ThreadedInDataHandler<PyIndataHandler> {
    * is called, when a stream stops, this pointer / handle will dangle.
    */
   PyIndataHandler(SmgrHandle mgr, py::function cb, py::function reset_callback)
-      : ThreadedInDataHandler(mgr),
-        cb(std::make_unique<py::function>(cb)),
-        reset_callback(std::make_unique<py::function>(reset_callback)) {
+      : ThreadedInDataHandler(mgr), _cb(cb), _reset_callback(reset_callback) {
     DEBUGTRACE_ENTER;
     /// Start should be called externally, as at constructor time no virtual
     /// functions should be called.
@@ -139,15 +139,24 @@ class PyIndataHandler : public ThreadedInDataHandler<PyIndataHandler> {
    *
    * @param daq Daq device, or nullptr in case no input stream is running.
    */
-  void reset(const Daq *daq) {
+  void reset(const Daq *daqi) {
     DEBUGTRACE_ENTER;
     if (_done) return;
-    try {
+    {
       py::gil_scoped_acquire acquire;
-      if (daq) {
-        (*reset_callback)(daq);
+      try {
+        py::object reset_callback = _reset_callback();
+        if (reset_callback.is_none()) {
+          DEBUGTRACE_PRINT("cb is none, weakref killed");
+          _done = true;
+          return;
+        }
+        if (daqi != nullptr) {
+          assert(reset_callback);
+          reset_callback(daqi);
         } else {
-        (*reset_callback)(py::none());
+          assert(reset_callback);
+          reset_callback(py::none());
         }
       } catch (py::error_already_set &e) {
         cerr << "*************** Error calling reset callback!\n";
@@ -158,53 +167,71 @@ class PyIndataHandler : public ThreadedInDataHandler<PyIndataHandler> {
         abort();
         /* throw std::runtime_error(e.what()); */
       } catch (std::exception &e) {
-      cerr << "Caught unknown exception in reset callback:" << e.what() << endl;
+        cerr << "Caught unknown exception in reset callback:" << e.what()
+             << endl;
         abort();
       }
     }
+  }
 
   /**
    * @brief Calls the Python callback method / function with a Numpy array of
    * stream data.
    */
   void inCallback(const DaqData &d) {
-
-    /* DEBUGTRACE_ENTER; */
+    // DEBUGTRACE_ENTER;
+    // cerr << "Thread ID: " << std::this_thread::get_id() << endl;
 
     using DataType = DataTypeDescriptor::DataType;
-    if (_done) return;
-
-    try {
+    if (_done) {
+      DEBUGTRACE_PRINT("Early stop, done");
+      return;
+    }
+    {
       py::gil_scoped_acquire acquire;
-      py::object bool_val;
+      try {
+        py::object py_bool;
+        py::object cb = _cb();
+        if (cb.is_none()) {
+          DEBUGTRACE_PRINT("cb is none, weakref killed");
+          _done = true;
+          return;
+        }
         switch (d.dtype) {
           case (DataType::dtype_int8): {
-          bool_val = (*cb)(getPyArrayNoCpy<int8_t>(d));
+            py_bool = cb(getPyArrayNoCpy<int8_t>(d));
           } break;
           case (DataType::dtype_int16): {
-          bool_val = (*cb)(getPyArrayNoCpy<int16_t>(d));
+            py_bool = cb(getPyArrayNoCpy<int16_t>(d));
           } break;
           case (DataType::dtype_int32): {
-          bool_val = (*cb)(getPyArrayNoCpy<int32_t>(d));
+            py_bool = cb(getPyArrayNoCpy<int32_t>(d));
           } break;
           case (DataType::dtype_fl32): {
-          bool_val = (*cb)(getPyArrayNoCpy<float>(d));
+            py_bool = cb(getPyArrayNoCpy<float>(d));
           } break;
           case (DataType::dtype_fl64): {
-          bool_val = (*cb)(getPyArrayNoCpy<double>(d));
+            py_bool = cb(getPyArrayNoCpy<double>(d));
           } break;
           default:
             throw std::runtime_error("BUG");
         }  // End of switch
 
-      bool res = bool_val.cast<bool>();
+        bool res = py_bool.cast<bool>();
         if (res == false) {
           DEBUGTRACE_PRINT("Setting callbacks to None")
           _done = true;
-        // cb = py::function(py::none());
-        // reset_callback = py::function(py::none());
-        cb.reset();
-        reset_callback.reset();
+
+          // By doing this, we remove the references, but in the mean time this
+          // might also trigger removing Python objects. Including itself, as
+          // there is no reference to it anymore. The consequence is that the
+          // current object might be destroyed from this thread. However, if we
+          // do not remove these references and in lasp_record.py finish() is
+          // not called, we end up with not-garbage collected recordings in
+          // memory. This is also not good. How can we force Python to not yet
+          // destroy this object?
+          // cb.reset();
+          // reset_callback.reset();
         }
       } catch (py::error_already_set &e) {
         cerr << "ERROR: Python raised exception from callback function: ";
@@ -219,11 +246,12 @@ class PyIndataHandler : public ThreadedInDataHandler<PyIndataHandler> {
              << endl;
         abort();
       }
-  }
+
+    }  // End of scope in which the GIL is acquired
+  }    // End of function
 };
 
 void init_datahandler(py::module &m) {
-
   /// The C++ class is PyIndataHandler, but for Python, it is called
   /// InDataHandler
   py::class_<PyIndataHandler> pyidh(m, "InDataHandler");

python_src/lasp/lasp_record.py

@@ -190,7 +190,7 @@ class Recording:
         When returning False, it will stop the stream.
 
         """
-        logger.debug(f"inCallback({adata})")
+        logger.debug(f"inCallback()")
         if self._stop():
             logger.debug("Stop flag set, early return in inCallback")
             # Stop flag is raised. We do not add any data anymore.
@@ -228,10 +228,10 @@ class Recording:
                         self._progressCallback(recstatus)
 
                 case RecordingState.AllDataStored:
-                    pass
+                    return False
 
                 case RecordingState.Finished:
-                    pass
+                    return False
 
             return True