#!/usr/bin/env python3.6 # -*- coding: utf-8 -*- """ Description: Read data from image stream and record sound at the same time """ import cv2 as cv from .lasp_atomic import Atomic from threading import Thread, Condition, Lock import numpy as np import time from .device import (RtAudio, DeviceInfo, DAQConfiguration, get_numpy_dtype_from_format_string, get_sampwidth_from_format_string) __all__ = ['AvType', 'AvStream'] video_x, video_y = 640, 480 class AvType: """Specificying the type of data, for adding and removing callbacks from the stream.""" audio_input = 1 audio_output = 2 video = 4 class AvStream: """Audio and video data stream, to which callbacks can be added for processing the data.""" def __init__(self, device: DeviceInfo, avtype: AvType, daqconfig: DAQConfiguration, video=None): """Open a stream for audio in/output and video input. For audio output, by default all available channels are opened for outputting data. Args: device: DeviceInfo for the audio device avtype: Type of stream. Input, output or duplex daqconfig: DAQConfiguration instance. If duplex mode flag is set, please make sure that output_device is None, as in that case the output config will be taken from the input device. video: """ self.daqconfig = daqconfig self._device = device self.avtype = avtype self.duplex_mode = daqconfig.duplex_mode self.monitor_gen = daqconfig.monitor_gen # Determine highest input channel number channelconfigs = daqconfig.en_input_channels self.channel_names = [] self.sensitivity = self.daqconfig.getSensitivities() if daqconfig.monitor_gen: assert self.duplex_mode self.channel_names.append('Generated signal') self.sensitivity = np.concatenate([np.array([1.]), self.sensitivity]) rtaudio_inputparams = None rtaudio_outputparams = None self.nframes_per_block = 2048 if self.duplex_mode or avtype == AvType.audio_output: rtaudio_outputparams = {'deviceid': device.index, # TODO: Add option to specify the number of output channels to use 'nchannels': 1, # device.outputchannels, 'firstchannel': 0} self.sampleformat = daqconfig.en_output_sample_format self.samplerate = int(daqconfig.en_output_rate) if avtype == AvType.audio_input or self.duplex_mode: for i, channelconfig in enumerate(channelconfigs): if channelconfig.channel_enabled: self.nchannels = i+1 self.channel_names.append(channelconfig.channel_name) rtaudio_inputparams = {'deviceid': device.index, 'nchannels': self.nchannels, 'firstchannel': firstchannel} # Here, we override the sample format in case of duplex mode. self.sampleformat = daqconfig.en_input_sample_format self.samplerate = int(daqconfig.en_input_rate) try: self._rtaudio = RtAudio() self.blocksize = self._rtaudio.openStream( rtaudio_outputparams, # Outputparams rtaudio_inputparams, # Inputparams self.sampleformat, # Sampleformat self.samplerate, self.nframes_per_block, # Buffer size in frames self._audioCallback) except Exception as e: raise RuntimeError(f'Could not initialize DAQ device: {str(e)}') # Fill in numpy data type, and sample width self.numpy_dtype = get_numpy_dtype_from_format_string( self.sampleformat) self.sampwidth = get_sampwidth_from_format_string( self.sampleformat) # Counters for the number of frames that have been coming in self._aframectr = Atomic(0) self._vframectr = Atomic(0) # Lock self._callbacklock = Lock() self._running = Atomic(False) self._running_cond = Condition() self._video = video self._video_started = Atomic(False) # Storage for callbacks, specified by type self._callbacks = { AvType.audio_input: [], AvType.audio_output: [], AvType.video: [] } # Possible, but long not tested: store video self._videothread = None def close(self): self._rtaudio.closeStream() self._rtaudio = None def nCallbacks(self): """Returns the current number of installed callbacks.""" return len(self._callbacks[AvType.audio_input]) + \ len(self._callbacks[AvType.audio_output]) + \ len(self._callbacks[AvType.video]) def addCallback(self, cb: callable, cbtype: AvType): """Add as stream callback to the list of callbacks.""" with self._callbacklock: outputcallbacks = self._callbacks[AvType.audio_output] if cbtype == AvType.audio_output and len(outputcallbacks) > 0: raise RuntimeError( 'Only one audio output callback can be allowed') if cb not in self._callbacks[cbtype]: self._callbacks[cbtype].append(cb) def removeCallback(self, cb, cbtype: AvType): with self._callbacklock: if cb in self._callbacks[cbtype]: self._callbacks[cbtype].remove(cb) def start(self): """Start the stream, which means the callbacks are called with stream data (audio/video)""" if self._running: raise RuntimeError('Stream already started') assert self._videothread is None self._running <<= True if self._video is not None: self._videothread = Thread(target=self._videoThread) self._videothread.start() else: self._video_started <<= True self._rtaudio.startStream() def _videoThread(self): cap = cv.VideoCapture(self._video) if not cap.isOpened(): cap.open() vframectr = 0 loopctr = 0 while self._running: ret, frame = cap.read() # print(frame.shape) if ret is True: if vframectr == 0: self._video_started <<= True with self._callbacklock: for cb in self._callbacks[AvType.video]: cb(frame, vframectr) vframectr += 1 self._vframectr += 1 else: loopctr += 1 if loopctr == 10: print('Error: no video capture!') time.sleep(0.2) cap.release() print('stopped videothread') def _audioCallback(self, indata, outdata, nframes, streamtime): """This is called (from a separate thread) for each audio block.""" self._aframectr += nframes with self._callbacklock: # Count the number of output callbacks. If no output callbacks are # present, and there should be output callbacks, we explicitly set # the output buffer to zero noutput_cb = len(self._callbacks[AvType.audio_output]) shouldhaveoutput = (self.avtype == AvType.audio_output or self.duplex_mode) if noutput_cb == 0 and shouldhaveoutput: outdata[:, :] = 0 # Loop over callbacks for cb in self._callbacks[AvType.audio_output]: try: cb(indata, outdata, self._aframectr()) except Exception as e: print(e) return 1 for cb in self._callbacks[AvType.audio_input]: try: cb(indata, outdata, self._aframectr()) except Exception as e: print(e) return 1 return 0 if self._running else 1 def stop(self): self._running <<= False with self._running_cond: self._running_cond.notify() if self._video: self._videothread.join() self._videothread = None self._aframectr <<= 0 self._vframectr <<= 0 self._video_started <<= False self._rtaudio.stopStream() def isRunning(self): return self._running() def hasVideo(self): return True if self._video is not None else False