Merged in develop
All checks were successful
Building, testing and releasing LASP if it has a tag / Build-Test-Ubuntu (push) Successful in 2m39s
Building, testing and releasing LASP if it has a tag / Release-Ubuntu (push) Has been skipped

This commit is contained in:
Anne de Jong 2024-01-10 17:31:59 +01:00
commit 6d5899c880
12 changed files with 388 additions and 51 deletions

View File

@ -1,5 +1,5 @@
cmake_minimum_required (VERSION 3.16)
project(LASP LANGUAGES C CXX VERSION 1.0)
project(LASP LANGUAGES C CXX VERSION 1.1)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED)

View File

@ -44,22 +44,13 @@ Daq::Daq(const DeviceInfo &devinfo, const DaqConfiguration &config)
: DaqConfiguration(config), DeviceInfo(devinfo) {
DEBUGTRACE_ENTER;
if (duplexMode()) {
if (neninchannels() == 0) {
throw rte("Duplex mode enabled, but no input channels enabled");
}
if (nenoutchannels() == 0) {
throw rte("Duplex mode enabled, but no output channels enabled");
}
}
if(!duplexMode() && monitorOutput) {
throw rte("Output monitoring only allowed when running in duplex mode");
throw rte("Duplex mode requires enabling both input and output channels. Please make sure at least one output channel is enabled, or disable hardware output loopback in DAQ configuration.");
}
if (!hasInternalOutputMonitor && monitorOutput) {
throw rte(
"Output monitor flag set, but device does not have output monitor");
"Output monitor flag set, but device does not have hardware output monitor.");
}
if (!config.match(devinfo)) {

View File

@ -35,12 +35,14 @@ DaqConfiguration::DaqConfiguration(const DeviceInfo &device) {
us i = 0;
for (auto &inch : inchannel_config) {
inch.name = "Unnamed input channel " + std::to_string(i);
inch.rangeIndex = device.prefInputRangeIndex;
i++;
}
i = 0;
for (auto &outch : outchannel_config) {
outch.name = "Unnamed output channel " + std::to_string(i);
outch.rangeIndex = device.prefOutputRangeIndex;
i++;
}

View File

@ -71,11 +71,23 @@ public:
* @brief Available ranges for the input, i.e. +/- 1V and/or +/- 10 V etc.
*/
dvec availableInputRanges;
/**
* @brief Its preffered range
* @brief Available ranges for the output, i.e. +/- 1V and/or +/- 10 V etc.
*/
dvec availableOutputRanges;
/**
* @brief Its preffered input range
*/
int prefInputRangeIndex = 0;
/**
* @brief Its preffered output range
*/
int prefOutputRangeIndex = 0;
/**
* @brief The number of input channels available for the device
*/
@ -125,13 +137,29 @@ public:
bool duplexModeForced = false;
/**
* @brief The physical quantity of the output signal. For 'normal' audio
* @brief Indicates whether the device is able to run in duplex mode. If false,
* devices cannot run in duplex mode, and the `duplexModeForced` flag is meaningless.
*/
bool hasDuplexMode = false;
/**
* @brief The physical quantity of the input signal from DAQ. For 'normal' audio
* interfaces, this is typically a 'number' between +/- full scale. For some
* real DAQ devices however, the input quantity corresponds to a physical signal,
* such a Volts.
*/
DaqChannel::Qty physicalInputQty = DaqChannel::Qty::Number;
/**
* @brief The physical quantity of the output signal from DAQ. For 'normal' audio
* devices, this is typically a 'number' between +/- full scale. For some
* devices however, the output quantity corresponds to a physical signal,
* real DAQ devices however, the input quantity corresponds to a physical signal,
* such a Volts.
*/
DaqChannel::Qty physicalOutputQty = DaqChannel::Qty::Number;
/**
* @brief String representation of DeviceInfo
*

View File

@ -99,6 +99,7 @@ void fillRtAudioDeviceInfo(DeviceInfoList &devinfolist) {
d.ninchannels = devinfo.inputChannels;
d.availableInputRanges = {1.0};
d.availableOutputRanges = {1.0};
RtAudioFormat formats = devinfo.nativeFormats;
if (formats & RTAUDIO_SINT8) {

View File

@ -68,6 +68,7 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) {
}
devinfo.physicalOutputQty = DaqChannel::Qty::Voltage;
devinfo.physicalInputQty = DaqChannel::Qty::Voltage;
devinfo.availableDataTypes.push_back(
DataTypeDescriptor::DataType::dtype_fl64);
@ -79,7 +80,9 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) {
devinfo.availableFramesPerBlock = {512, 1024, 2048, 4096, 8192};
devinfo.availableInputRanges = {1.0, 10.0};
devinfo.availableOutputRanges = {10.0};
devinfo.prefInputRangeIndex = 0;
devinfo.prefOutputRangeIndex = 0;
devinfo.ninchannels = 4;
devinfo.noutchannels = 1;
@ -90,6 +93,7 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) {
devinfo.hasInternalOutputMonitor = true;
devinfo.hasDuplexMode = true;
devinfo.duplexModeForced = true;
// Finally, this devinfo is pushed back in list

View File

@ -29,6 +29,9 @@ void init_deviceinfo(py::module& m) {
devinfo.def_readonly("availableInputRanges",
&DeviceInfo::availableInputRanges);
devinfo.def_readonly("prefInputRangeIndex", &DeviceInfo::prefInputRangeIndex);
devinfo.def_readonly("availableOutputRanges",
&DeviceInfo::availableOutputRanges);
devinfo.def_readonly("prefOutputRangeIndex", &DeviceInfo::prefOutputRangeIndex);
devinfo.def_readonly("ninchannels", &DeviceInfo::ninchannels);
devinfo.def_readonly("noutchannels", &DeviceInfo::noutchannels);
@ -36,7 +39,10 @@ void init_deviceinfo(py::module& m) {
devinfo.def_readonly("hasInputACCouplingSwitch",
&DeviceInfo::hasInputACCouplingSwitch);
devinfo.def_readonly("hasDuplexMode", &DeviceInfo::hasDuplexMode);
devinfo.def_readonly("duplexModeForced", &DeviceInfo::duplexModeForced);
devinfo.def_readonly("hasInternalOutputMonitor", &DeviceInfo::hasInternalOutputMonitor);
devinfo.def_readonly("physicalInputQty", &DeviceInfo::physicalInputQty);
devinfo.def_readonly("physicalOutputQty", &DeviceInfo::physicalOutputQty);
}

View File

@ -5,7 +5,7 @@ requires-python = ">=3.10"
description = "Library for Acoustic Signal Processing"
license = { "file" = "LICENSE" }
authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }]
version = "1.0.4"
version = "1.3.1"
keywords = ["DSP", "DAQ", "Signal processing"]

View File

@ -209,6 +209,7 @@ class FilterBankDesigner:
Returns:
h: Linear filter transfer function [-]
"""
fs = self.fs
fir = self.createFirFilter(fs, x)
# Decimated sampling frequency [Hz]

View File

@ -62,8 +62,9 @@ class Qty:
name: str
# I.e.: Pascal
unit_name: str
# I.e.: Pa
# I.e.: -, Pa, V
unit_symb: str
# I.e.: ('dB SPL') <== tuple of possible level units
level_unit: object
# Contains a tuple of possible level names, including its reference value.
@ -92,6 +93,18 @@ class Qty:
"""
return self.cpp_enum.value
@property
def unit_symb_eq(self):
"""Unit symbol to be used in equations
Returns:
String: V, Pa, 1,
"""
if self.unit_symb != '-':
return self.unit_symb
else:
return '1'
@unique

View File

@ -1,7 +1,6 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from __future__ import annotations
"""!
Author: J.A. de Jong - ASCEE
@ -11,18 +10,21 @@ The ASCEE hdf5 measurement file format contains the following fields:
- Attributes:
'version': If not given, version 1 is assumed. For version 1, measurement data
is assumed to be acoustic data.
'samplerate': The audio data sample rate in Hz.
'nchannels': The number of audio channels in the file
'LASP_VERSION_MAJOR': int The major version of LASP which which the recording has been performed.
'LASP_VERSION_MINOR': int The minor version of LASP which which the recording has been performed.
'samplerate': The audio data sample rate in Hz [float]
'nchannels': The number of audio channels in the file List[float]
'sensitivity': (Optionally) the stored sensitivity of the record channels.
This can be a single value, or a list of sensitivities for
each channel. Both representations are allowed.
each channel. Both representations are allowed. List[float]
For measurement files of LASP < v1.0
'qtys' : (Optionally): list of quantities that is recorded for each channel',
if this array is not found. Quantities are defaulted to 'Number / Full scale'
'type_int': A specified measurement type that can be used programmatically. It can be read out as an enumeration variant of type "MeasurementType". See code below of implemented measurement types.
For measurement files of LASP >= 1.0
- Datasets:
@ -32,7 +34,7 @@ block index, the second axis the sample number and the third axis is the
channel number. The data type is either int16, int32 or float64 / float32. If
raw data is stored as integer values (int16, int32), the actual values should
be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the
corresponding 'number' lies between -1.0 and 1.0.
corresponding 'number' lies between -1.0 and 1.0. To stay backwards-compatible, the dataset is always called 'audio' despite it being possible that other types of data is stored in the dataset (such as voltages, accelerations etc).
'video': 4-dimensional array of video frames. The first index is the frame
number, the second the x-value of the pixel and the third is the
@ -44,10 +46,16 @@ The video dataset can possibly be not present in the data.
"""
__all__ = ["Measurement", "scaleBlockSens"]
__all__ = ["Measurement", "scaleBlockSens", "MeasurementType"]
from contextlib import contextmanager
from weakref import WeakValueDictionary
import h5py as h5
import uuid
import pathlib
import glob
import itertools
import numpy as np
from enum import Enum, unique
from .lasp_config import LASP_NUMPY_FLOAT_TYPE
from scipy.io import wavfile
import os, time, wave, logging
@ -57,6 +65,32 @@ from .lasp_cpp import Window, DaqChannel, AvPowerSpectra
from typing import List
from functools import lru_cache
# Measurement file extension
MEXT = 'h5'
DOTMEXT = f'.{MEXT}'
@unique
class MeasurementType(Enum):
"""
Measurement flags related to the measurement. Stored as bit flags in the measurement file. This is for possible changes in the API later.
"""
# Not specific measurement type
NotSpecific = 0
# Measurement serves as an insertion loss reference measurement
ILReference = 1 << 0
# Measurement is general calibration measurement (to calibrate sensor in a certain way)
CALGeneral = 1 << 1
# Measurement serves as impedance tube calibration (short tube case / ref. plane at origin)
muZCalShort = 1 << 2
# Measurement serves as impedance tube calibration (long tube case)
muZCalLong = 1 << 3
def getSampWidth(dtype):
"""Returns the width of a single sample in **bytes**.
@ -188,17 +222,19 @@ class Measurement:
"""Provides access to measurement data stored in the h5 measurement file
format."""
# Store a dict of open measurements, with uuid string as a key. We store them as a weak ref.
uuid_s = WeakValueDictionary()
def __init__(self, fn):
"""Initialize a Measurement object based on the filename."""
if ".h5" not in fn:
fn += ".h5"
# Add extension if tried to open without exension
if DOTMEXT not in fn:
fn += DOTMEXT
# Full filepath
self.fn = fn
# Base filename
self.fn_base = os.path.split(fn)[1]
# Open the h5 file in read-plus mode, to allow for changing the
# measurement comment.
with h5.File(fn, "r") as f:
@ -222,9 +258,39 @@ class Measurement:
self.version_major = f.attrs["LASP_VERSION_MAJOR"]
self.version_minor = f.attrs["LASP_VERSION_MINOR"]
except KeyError:
# No version information stored
self.version_major = 0
self.version_minor = 1
try:
# Try to catch UUID (Universally Unique IDentifier)
self._UUID = f.attrs['UUID']
# Flag indicating we have to add a new UUID
create_new_uuid = False
except KeyError:
create_new_uuid = True
try:
# UUID of the reference measurement. Should be stored as
# a lists of tuples, where each tuple is a combination of (<MeasurementType.value>, <uuid_string>, <last_filename>).
# The last filename is a filename that *probably* is the reference measurement with
# given UUID. If it is not, we will search for it in the same directory as `this` measurement.
# If we cannot find it there, we will give up, and remove the field corresponding to this reference measurement type.
refMeas_list = f.attrs['refMeas']
# Build a tuple string from it
self._refMeas = {}
for (key, val, name) in refMeas_list:
self._refMeas[MeasurementType(int(key))] = (val, name)
except KeyError:
self._refMeas = {}
try:
self._type_int = f.attrs['type_int']
except KeyError:
self._type_int = 0
# Due to a previous bug, the channel names were not stored
# consistently, i.e. as 'channel_names' and later camelcase.
try:
@ -282,20 +348,202 @@ class Measurement:
f"Physical quantity data not available in measurement file. Assuming {SIQtys.default}"
)
def setAttribute(self, atrname, value):
if create_new_uuid:
# Create and store a random UUID based on *now* and store it forever
# inside of the Measurement file
self.genNewUUID()
else:
if self.UUID in Measurement.uuid_s.keys():
raise RuntimeError(f"Measurement '{self.name}' is already opened. Cannot open measurement twice. Note: this error can happen when measurements are manually copied.")
# Store weak reference to 'self' in list of UUID's. They are removed when no file is open anymore
Measurement.uuid_s[self._UUID] = self
def rename(self, newname: str):
"""
Try to rename the measurement file.
Args:
newname: New name, with or without extension
"""
_ , ext = os.path.splitext(newname)
# Add proper extension if new name is given without extension.
if ext != DOTMEXT:
newname = newname + DOTMEXT
# Folder, Base filename + extension
folder, _ = os.path.split(self.fn)
newname_full = str(pathlib.Path(folder) / newname)
os.rename(self.fn, newname_full)
self.fn = newname_full
def genNewUUID(self):
"""
Create new UUID for measurement and store in file.
"""
self.setAttribute('UUID', str(uuid.uuid1()))
@property
def UUID(self):
"""
Universally unique identifier
"""
return self._UUID
def getRefMeas(self, mtype: MeasurementType):
"""
Return corresponding reference measurement, if configured and can be found. If the reference
measurement is currently not open, it tries to open it by traversing other measurement
files in the current directory. Throws a runtime error in case the reference measurement cannot be found.
Throws a ValueError when the reference measurement is not configured.
"""
# See if we can find the UUID for the required measurement type
try:
required_uuid, possible_name = self._refMeas[mtype]
except KeyError:
raise ValueError(f"No reference measurement configured for '{self.name}'")
m = None
# Try to find it in the dictionary of of open measurements
if required_uuid in Measurement.uuid_s.keys():
m = Measurement.uuid_s[required_uuid]
logging.info(f'Returned reference measurement {m.name} from list of open measurements')
# Not found in list of openend measurements. See if we can open it using its last stored file name we know of
if m is None:
try:
m = Measurement(possible_name)
if m.UUID == required_uuid:
logging.info(f'Opened reference measurement {m.name} by name')
except Exception as e:
logging.error(f'Could not find reference measurement using file name: {possible_name}')
# Last resort, see if we can find the right measurement in the same folder
if m is None:
try:
folder, _ = os.path.split(self.fn)
m = Measurement.fromFolderWithUUID(required_uuid, folder, skip=[self.name])
logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement')
# Update the measurement file name in the list, such that next time it
# can be opened just by its name.
self.setRefMeas(m)
except:
logging.error("Could not find the reference measurement. Is it deleted?")
# Well, we found it. Now make sure the reference measurement actually has the right type (User could have marked it as a NotSpecific for example in the mean time).
if m is not None:
if m.measurementType() != mtype:
m.removeRefMeas(mtype)
raise RuntimeError(f"Reference measurement for {self.name} is not a proper reference (anymore).")
# Whow, we passed all security checks, here we go!
return m
else:
# Nope, not there.
raise RuntimeError(f"Could not find the reference measurement for '{self.name}'. Is it deleted?")
def removeRefMeas(self, mtype: MeasurementType):
"""
Remove an existing reference measurement of specified type from this measurement. Silently ignores this
action if no reference measurement of this type is configured.
"""
try:
del self._refMeas[mtype]
self.__storeReafMeas()
except KeyError:
pass
def __storeReafMeas(self):
"""
Internal method that syncs the dictionary of reference methods with the backing HDF5 file
"""
with self.file("r+") as f:
# Update attribute in file. Stored as a flat list of string tuples:
# [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...]
reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items())
# print(reflist)
f.attrs['refMeas'] = reflist
def setRefMeas(self, m: Measurement):
"""
Set a reference measurement for the given measurement. If this measurement is already
a reference measurement, the previous reference measurement type is overwritten, such that there is
only one measurement that is the reference of a certain 'MeasurementType'
"""
mtype = m.measurementType()
if mtype == MeasurementType.NotSpecific:
raise ValueError('Measurement to be set as reference is not a reference measurement')
self._refMeas[mtype] = (m.UUID, m.name)
self.__storeReafMeas()
@staticmethod
def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]):
"""
Returns Measurement object from a given UUID string. It first tries to find whether there
is an uuid in the static list of weak references. If not, it will try to open files in
the current file path.
"""
for fn in glob.glob(str(pathlib.Path(folder)) + f'/*{DOTMEXT}'):
# Do not try to open this file in case it is in the 'skip' list.
if len(list(filter(lambda a: a in fn, skip))) > 0:
continue
try:
m = Measurement(fn)
if m.UUID == uuid_str:
# Update 'last_fn' attribute in dict of stored reference measurements
return m
except Exception as e:
logging.error(f'Possible measurement file {fn} returned error {e} when opening.')
raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.')
def setAttribute(self, attrname: str, value):
"""
Set an attribute in the measurement file, and keep a local copy in
memory for efficient accessing.
Args:
attrname: name of attribute, a string
value: the value. Should be anything that can be stored as an attribute in HDF5.
"""
with self.file("r+") as f:
# Update comment attribute in the file
f.attrs[atrname] = value
setattr(self, "_" + atrname, value)
f.attrs[attrname] = value
setattr(self, "_" + attrname, value)
def isType(self, type_: MeasurementType):
"""
Returns True when a measurement is flagged as being of a certaint "MeasurementType"
"""
if (type_.value & self._type_int):
return True
elif type_.value == self._type_int == 0:
return True
return False
def setType(self, type_: MeasurementType):
"""
Set the measurement type to given type
"""
self.setAttribute('type_int', type_.value)
def measurementType(self):
"""
Returns type of measurement
"""
return MeasurementType(self._type_int)
@property
def name(self):
"""Returns filename base without extension."""
return os.path.splitext(self.fn_base)[0]
_, fn = os.path.split(self.fn)
return os.path.splitext(fn)[0]
@property
def channelNames(self):
@ -303,12 +551,18 @@ class Measurement:
@channelNames.setter
def channelNames(self, newchnames):
"""
Returns list of the names of the channels
"""
if len(newchnames) != self.nchannels:
raise RuntimeError("Invalid length of new channel names")
self.setAttribute("channelNames", newchnames)
@property
def channelConfig(self):
"""
Returns list of current channel configuration data.
"""
chcfg = []
for chname, sens, qty in zip(self.channelNames, self.sensitivity, self.qtys):
ch = DaqChannel()
@ -321,6 +575,18 @@ class Measurement:
@channelConfig.setter
def channelConfig(self, chcfg: List[DaqChannel]):
"""
Set new channel configuration from list of DaqChannel objects.
Use cases:
- Update channel types, sensitivities etc.
Args:
chchfg: New channel configuration
"""
if len(chcfg) != self.nchannels:
raise RuntimeError("Invalid number of channels")
chname = []
sens = []
qtys = []
@ -711,6 +977,28 @@ class Measurement:
wavfile.write(fn, int(self.samplerate), data.astype(newtype))
@staticmethod
def fromFile(fn):
"""
Try to open measurement from a given file name. First checks
whether the measurement is already open. Otherwise it might
happen that a Measurement object is created twice for the same backing file, which we do not allow.
"""
# See if the base part of the filename is referring to a file that is already open
with h5.File(fn, 'r') as f:
try:
theuuid = f.attrs['UUID']
except KeyError:
# No UUID stored in measurement. This is an old measurement that did not have UUID's
# We create a new UUID here such that the file is opened from the filesystem
# anyhow.
theuuid = str(uuid.uuid1())
if theuuid in Measurement.uuid_s.keys():
return Measurement.uuid_s[theuuid]
return Measurement(fn)
@staticmethod
def fromtxt(
fn,
@ -744,10 +1032,11 @@ class Measurement:
raise ValueError(f"File {fn} does not exist.")
if timestamp is None:
timestamp = os.path.getmtime(fn)
if mfn is None:
mfn = os.path.splitext(fn)[0] + ".h5"
mfn = os.path.splitext(fn)[0] + DOTMEXT
else:
mfn = os.path.splitext(mfn)[0] + ".h5"
mfn = os.path.splitext(mfn)[0] + DOTMEXT
dat = np.loadtxt(fn, skiprows=skiprows, delimiter=delimiter)
if firstcoltime:
@ -818,8 +1107,9 @@ class Measurement:
force: If True, overwrites existing files with specified `mfn`
name.
"""
if os.path.splitext(mfn)[1] != ".h5":
mfn += ".h5"
if os.path.splitext(mfn)[1] != DOTMEXT:
mfn += DOTMEXT
if os.path.exists(mfn) and not force:
raise ValueError(f"File {mfn} already exist.")
if timestamp is None:
@ -850,8 +1140,6 @@ class Measurement:
if len(qtys) != nchannels:
raise RuntimeError("Illegal length of qtys list given")
qtyvals = [qty.value for qty in qtys]
with h5.File(mfn, "w") as hf:
hf.attrs["samplerate"] = samplerate
hf.attrs["sensitivity"] = sensitivity
@ -860,7 +1148,7 @@ class Measurement:
hf.attrs["nchannels"] = nchannels
# Add physical quantity indices
hf.attrs["qtys_enum_idx"] = [qtyval.toInt() for qtyval in qtyvals]
hf.attrs['qtys_enum_idx'] = [qty.toInt() for qty in qtys]
# Add channel names in case given
if channelNames is not None:
@ -916,3 +1204,4 @@ class Measurement:
ad[0] = data
return Measurement(newfn)

View File

@ -9,6 +9,7 @@ import numpy as np
from .lasp_atomic import Atomic
from .lasp_cpp import InDataHandler, StreamMgr
from .lasp_version import LASP_VERSION_MAJOR, LASP_VERSION_MINOR
import uuid
@dataclasses.dataclass
@ -139,6 +140,7 @@ class Recording:
f.attrs["blocksize"] = blocksize
f.attrs["sensitivity"] = [ch.sensitivity for ch in in_ch]
f.attrs["channelNames"] = [ch.name for ch in in_ch]
f.attrs["UUID"] = str(uuid.uuid1())
# Add the start delay here, as firstFrames() is called right after the
# constructor is called. time.time() returns a floating point