From c610c6350d4a0b9bde5de45b0829b7a331d03060 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Mon, 18 Dec 2023 12:37:46 +0100 Subject: [PATCH 01/12] Implemented reference measurements, and renaming as method in Measurement object. --- CMakeLists.txt | 2 +- pyproject.toml | 2 +- python_src/lasp/lasp_measurement.py | 262 ++++++++++++++++++++++++++-- 3 files changed, 248 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43173bf..d28253b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 3.16) -project(LASP LANGUAGES C CXX VERSION 1.0) +project(LASP LANGUAGES C CXX VERSION 1.1) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED) diff --git a/pyproject.toml b/pyproject.toml index a2aface..eba62a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.10" description = "Library for Acoustic Signal Processing" license = { "file" = "LICENSE" } authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }] -version = "1.0.1" +version = "1.1.0" keywords = ["DSP", "DAQ", "Signal processing"] diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index f798577..b2e9d85 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -11,18 +11,21 @@ The ASCEE hdf5 measurement file format contains the following fields: - Attributes: -'version': If not given, version 1 is assumed. For version 1, measurement data -is assumed to be acoustic data. -'samplerate': The audio data sample rate in Hz. -'nchannels': The number of audio channels in the file +'LASP_VERSION_MAJOR': int The major version of LASP which which the recording has been performed. +'LASP_VERSION_MINOR': int The minor version of LASP which which the recording has been performed. + +'samplerate': The audio data sample rate in Hz [float] +'nchannels': The number of audio channels in the file List[float] 'sensitivity': (Optionally) the stored sensitivity of the record channels. This can be a single value, or a list of sensitivities for - each channel. Both representations are allowed. + each channel. Both representations are allowed. List[float] For measurement files of LASP < v1.0 'qtys' : (Optionally): list of quantities that is recorded for each channel', if this array is not found. Quantities are defaulted to 'Number / Full scale' +'type_int': A specified measurement type that can be used programmatically. It can be read out as an enumeration variant of type "MeasurementType". See code below of implemented measurement types. + For measurement files of LASP >= 1.0 - Datasets: @@ -32,7 +35,7 @@ block index, the second axis the sample number and the third axis is the channel number. The data type is either int16, int32 or float64 / float32. If raw data is stored as integer values (int16, int32), the actual values should be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the -corresponding 'number' lies between -1.0 and 1.0. +corresponding 'number' lies between -1.0 and 1.0. To stay backwards-compatible, the dataset is always called 'audio' despite it being possible that other types of data is stored in the dataset (such as voltages, accelerations etc). 'video': 4-dimensional array of video frames. The first index is the frame number, the second the x-value of the pixel and the third is the @@ -44,10 +47,16 @@ The video dataset can possibly be not present in the data. """ -__all__ = ["Measurement", "scaleBlockSens"] +__all__ = ["Measurement", "scaleBlockSens", "MeasurementType"] from contextlib import contextmanager +from weakref import WeakValueDictionary import h5py as h5 +import uuid +import pathlib +import glob +import itertools import numpy as np +from enum import Enum, unique from .lasp_config import LASP_NUMPY_FLOAT_TYPE from scipy.io import wavfile import os, time, wave, logging @@ -57,6 +66,32 @@ from .lasp_cpp import Window, DaqChannel, AvPowerSpectra from typing import List from functools import lru_cache +# Measurement file extension +MEXT = 'h5' +DOTMEXT = f'.{MEXT}' + + +@unique +class MeasurementType(Enum): + """ + Measurement flags related to the measurement. Stored as bit flags in the measurement file. This is for possible changes in the API later. + """ + + # Not specific measurement type + NotSpecific = 0 + + # Measurement serves as an insertion loss reference measurement + ILReference = 1 << 0 + + # Measurement is general calibration measurement (to calibrate sensor in a certain way) + CALGeneral = 1 << 1 + + # Measurement serves as impedance tube calibration (short tube case / ref. plane at origin) + muZCalShort = 1 << 2 + + # Measurement serves as impedance tube calibration (long tube case) + muZCalLong = 1 << 3 + def getSampWidth(dtype): """Returns the width of a single sample in **bytes**. @@ -188,16 +223,21 @@ class Measurement: """Provides access to measurement data stored in the h5 measurement file format.""" + # Store a dict of open measurements, with uuid string as a key. We store them as a weak ref. + uuid_s = WeakValueDictionary() + def __init__(self, fn): """Initialize a Measurement object based on the filename.""" - if ".h5" not in fn: - fn += ".h5" + + # Add extension if tried to open without exension + if DOTMEXT not in fn: + fn += DOTMEXT # Full filepath self.fn = fn - # Base filename - self.fn_base = os.path.split(fn)[1] + # Folder, Base filename + extension + self.folder, self.fn_base = os.path.split(fn) # Open the h5 file in read-plus mode, to allow for changing the # measurement comment. @@ -222,9 +262,39 @@ class Measurement: self.version_major = f.attrs["LASP_VERSION_MAJOR"] self.version_minor = f.attrs["LASP_VERSION_MINOR"] except KeyError: + # No version information stored self.version_major = 0 self.version_minor = 1 + try: + # Try to catch UUID (Universally Unique IDentifier) + self._UUID = f.attrs['UUID'] + # Flag indicating we have to add a new UUID + create_new_uuid = False + except KeyError: + create_new_uuid = True + + try: + # UUID of the reference measurement. Should be stored as + # a lists of tuples, where each tuple is a combination of (, , ). + + # The last filename is a filename that *probably* is the reference measurement with + # given UUID. If it is not, we will search for it in the same directory as `this` measurement. + # If we cannot find it there, we will give up, and remove the field corresponding to this reference measurement type. + refMeas_list = f.attrs['refMeas'] + + # Build a tuple string from it + self._refMeas = {} + for (key, val, name) in refMeas_list: + self._refMeas[MeasurementType(int(key))] = (val, name) + except KeyError: + self._refMeas = {} + + try: + self._type_int = f.attrs['type_int'] + except KeyError: + self._type_int = 0 + # Due to a previous bug, the channel names were not stored # consistently, i.e. as 'channel_names' and later camelcase. try: @@ -282,15 +352,155 @@ class Measurement: f"Physical quantity data not available in measurement file. Assuming {SIQtys.default}" ) - def setAttribute(self, atrname, value): + if create_new_uuid: + # Create and store a random UUID based on *now* and store it forever + # inside of the Measurement file + self.genNewUUID() + else: + if self.UUID in Measurement.uuid_s.keys(): + raise RuntimeError(f"Measurement '{self.name}' is already opened. Cannot open measurement twice. Note: this error can happen when measurements are manually copied.") + + # Store weak reference to 'self' in list of UUID's. They are removed when no file is open anymore + Measurement.uuid_s[self._UUID] = self + + def rename(self, newname: str): + """ + Try to rename the measurement file. + """ + _ , ext = os.path.splitext(newname) + # Add proper extension if new name is given without extension. + if ext != DOTMEXT: + newname = newname + DOTMEXT + + newname_full = str(pathlib.Path(self.folder) / newname ) + os.rename(self.fn, newname_full) + + def genNewUUID(self): + """ + Create new UUID for measurement and store in file. + """ + self.setAttribute('UUID', str(uuid.uuid4())) + + @property + def UUID(self): + """ + Universally unique identifier + """ + return self._UUID + + def refMeas(self, mtype: MeasurementType): + """ + Return corresponding reference measurement, if configured and can be found. If the reference + measurement is currently not open, it tries to open it by traversing other measurement + files in the current directory. Throws a runtime error in case the reference measurement cannot be found. + Throws a ValueError when the reference measurement is not configured. + """ + + # See if we can find the UUID for the required measurement type + try: + required_uuid, possible_name = self._refMeas[mtype] + except KeyError: + raise ValueError(f"No reference measurement configured for '{self.name}'") + + # Try to find it in the dictionary of of open measurements + if required_uuid in Measurement.uuid_s.keys(): + m = Measurement.uuid_s[required_uuid] + logging.info(f'Returned reference measurement {m.name} from list of open measurements') + return m + + # See if we can open it using its last stored file name + try: + m = Measurement(possible_name) + if m.UUID == required_uuid: + logging.info(f'Opened reference measurement {m.name} by name') + return m + except Exception as e: + logging.error(f'Could not find reference measurement using file name: {possible_name}') + + # Last resort, see if we can find the measurement in the same folder + m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name]) + logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement') + # Update the measurement file name in the list, such that next time it + # can be opened just by its name. + self.setRefMeas(m) + return m + + raise RuntimeError("Could not find reference measurement") + + def setRefMeas(self, m: Measurement): + """ + Set a reference measurement for the given measurement. If this measurement is already + a reference measurement, the previous reference measurement type is overwritten, such that there is + only one measurement that is the reference of a certain 'MeasurementType' + """ + mtype = m.measurementType() + if mtype == MeasurementType.NotSpecific: + raise ValueError('Measurement to be set as reference is not a reference measurement') + + self._refMeas[mtype] = (m.UUID, m.name) + with self.file("r+") as f: + # Update attribute in file. Stored as a flat list of string tuples: + # [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...] + reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items()) + # print(reflist) + f.attrs['refMeas'] = reflist + + @staticmethod + def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]): + """ + Returns Measurement object from a given UUID string. It first tries to find whether there + is an uuid in the static list of weak references. If not, it will try to open files in + the current file path. + """ + for fn in glob.glob(str(pathlib.Path(folder)) + f'/*{DOTMEXT}'): + # Do not try to open this file in case it is in the 'skip' list. + if len(list(filter(lambda a: a in fn, skip))) > 0: + continue + + try: + m = Measurement(fn) + if m.UUID == uuid_str: + # Update 'last_fn' attribute in dict of stored reference measurements + return m + except Exception as e: + logging.error(f'Possible measurement file {fn} returned error {e} when opening.') + + raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.') + + def setAttribute(self, attrname, value): """ Set an attribute in the measurement file, and keep a local copy in memory for efficient accessing. + + Args: + atrname """ with self.file("r+") as f: # Update comment attribute in the file - f.attrs[atrname] = value - setattr(self, "_" + atrname, value) + f.attrs[attrname] = value + setattr(self, "_" + attrname, value) + + def isType(self, type_: MeasurementType): + """ + Returns True when a measurement is flagged as being of a certaint "MeasurementType" + """ + if (type_.value & self._type_int): + return True + elif type_.value == self._type_int == 0: + return True + return False + + def setType(self, type_: MeasurementType): + """ + Set the measurement type to given type + """ + self.setAttribute('type_int', type_.value) + + def measurementType(self): + """ + Returns type of measurement + """ + return MeasurementType(self._type_int) @property def name(self): @@ -303,12 +513,18 @@ class Measurement: @channelNames.setter def channelNames(self, newchnames): + """ + Returns list of the names of the channels + """ if len(newchnames) != self.nchannels: raise RuntimeError("Invalid length of new channel names") self.setAttribute("channelNames", newchnames) @property def channelConfig(self): + """ + Returns list of current channel configuration data. + """ chcfg = [] for chname, sens, qty in zip(self.channelNames, self.sensitivity, self.qtys): ch = DaqChannel() @@ -321,6 +537,18 @@ class Measurement: @channelConfig.setter def channelConfig(self, chcfg: List[DaqChannel]): + """ + Set new channel configuration from list of DaqChannel objects. + + Use cases: + - Update channel types, sensitivities etc. + + Args: + chchfg: New channel configuration + """ + if len(chcfg) != self.nchannels: + raise RuntimeError("Invalid number of channels") + chname = [] sens = [] qtys = [] @@ -818,8 +1046,9 @@ class Measurement: force: If True, overwrites existing files with specified `mfn` name. """ - if os.path.splitext(mfn)[1] != ".h5": - mfn += ".h5" + + if os.path.splitext(mfn)[1] != DOTMEXT: + mfn += DOTMEXT if os.path.exists(mfn) and not force: raise ValueError(f"File {mfn} already exist.") if timestamp is None: @@ -916,3 +1145,4 @@ class Measurement: ad[0] = data return Measurement(newfn) + From 8c7dbed606b7e0ce53a0b8e450e7496d593f6402 Mon Sep 17 00:00:00 2001 From: Thijs Hekman Date: Tue, 19 Dec 2023 11:21:13 +0100 Subject: [PATCH 02/12] Bugfix in qty input of Measurement.fromnpy() --- src/lasp/lasp_measurement.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lasp/lasp_measurement.py b/src/lasp/lasp_measurement.py index 899bb74..3aeeb27 100644 --- a/src/lasp/lasp_measurement.py +++ b/src/lasp/lasp_measurement.py @@ -834,7 +834,6 @@ class Measurement: if len(qtys) != nchannels: raise RuntimeError("Illegal length of qtys list given") - qtyvals = [qty.value for qty in qtys] with h5.File(mfn, 'w') as hf: hf.attrs['samplerate'] = samplerate @@ -844,7 +843,7 @@ class Measurement: hf.attrs['nchannels'] = nchannels # Add physical quantity indices - hf.attrs['qtys_enum_idx'] = [qtyval.toInt() for qtyval in qtyvals] + hf.attrs['qtys_enum_idx'] = [qty.toInt() for qty in qtys] # Add channel names in case given if channelNames is not None: From 87283e4aba146757feea3417de5037ed4bd3718f Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Tue, 19 Dec 2023 12:26:22 +0100 Subject: [PATCH 03/12] Changed pure random UUID of measurement to time-based UUID --- python_src/lasp/lasp_measurement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index b2e9d85..61a4ec4 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -379,7 +379,7 @@ class Measurement: """ Create new UUID for measurement and store in file. """ - self.setAttribute('UUID', str(uuid.uuid4())) + self.setAttribute('UUID', str(uuid.uuid1())) @property def UUID(self): From 98d4e8dad20254f03e43c9048b8e8c1678dfd0f8 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Tue, 19 Dec 2023 12:27:44 +0100 Subject: [PATCH 04/12] More checks on refMeas. Renamed method refMeas to getRefMeas. Added removeRefMeas. Stabilized API? --- python_src/lasp/lasp_measurement.py | 94 +++++++++++++++++++---------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index 61a4ec4..6b70fd3 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -372,7 +372,7 @@ class Measurement: if ext != DOTMEXT: newname = newname + DOTMEXT - newname_full = str(pathlib.Path(self.folder) / newname ) + newname_full = str(pathlib.Path(self.folder) / newname) os.rename(self.fn, newname_full) def genNewUUID(self): @@ -388,7 +388,7 @@ class Measurement: """ return self._UUID - def refMeas(self, mtype: MeasurementType): + def getRefMeas(self, mtype: MeasurementType): """ Return corresponding reference measurement, if configured and can be found. If the reference measurement is currently not open, it tries to open it by traversing other measurement @@ -402,30 +402,66 @@ class Measurement: except KeyError: raise ValueError(f"No reference measurement configured for '{self.name}'") + m = None # Try to find it in the dictionary of of open measurements if required_uuid in Measurement.uuid_s.keys(): m = Measurement.uuid_s[required_uuid] logging.info(f'Returned reference measurement {m.name} from list of open measurements') - return m - # See if we can open it using its last stored file name - try: - m = Measurement(possible_name) - if m.UUID == required_uuid: - logging.info(f'Opened reference measurement {m.name} by name') - return m - except Exception as e: - logging.error(f'Could not find reference measurement using file name: {possible_name}') + # Not found in list of openend measurements. See if we can open it using its last stored file name we know of + if m is None: + try: + m = Measurement(possible_name) + if m.UUID == required_uuid: + logging.info(f'Opened reference measurement {m.name} by name') + except Exception as e: + logging.error(f'Could not find reference measurement using file name: {possible_name}') - # Last resort, see if we can find the measurement in the same folder - m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name]) - logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement') - # Update the measurement file name in the list, such that next time it - # can be opened just by its name. - self.setRefMeas(m) - return m - - raise RuntimeError("Could not find reference measurement") + # Last resort, see if we can find the right measurement in the same folder + if m is None: + try: + m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name]) + logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement') + # Update the measurement file name in the list, such that next time it + # can be opened just by its name. + self.setRefMeas(m) + except: + logging.error("Could not find the reference measurement. Is it deleted?") + + # Well, we found it. Now make sure the reference measurement actually has the right type (User could have marked it as a NotSpecific for example in the mean time). + if m is not None: + if m.measurementType() != mtype: + m.removeRefMeas(mtype) + raise RuntimeError(f"Reference measurement for {self.name} is not a proper reference (anymore).") + + # Whow, we passed all security checks, here we go! + return m + else: + # Nope, not there. + raise RuntimeError(f"Could not find the reference measurement for '{self.name}'. Is it deleted?") + + def removeRefMeas(self, mtype: MeasurementType): + """ + Remove an existing reference measurement of specified type from this measurement. Silently ignores this + action if no reference measurement of this type is configured. + + """ + try: + del self._refMeas[mtype] + self.__storeReafMeas() + except KeyError: + pass + + def __storeReafMeas(self): + """ + Internal method that syncs the dictionary of reference methods with the backing HDF5 file + """ + with self.file("r+") as f: + # Update attribute in file. Stored as a flat list of string tuples: + # [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...] + reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items()) + # print(reflist) + f.attrs['refMeas'] = reflist def setRefMeas(self, m: Measurement): """ @@ -438,12 +474,7 @@ class Measurement: raise ValueError('Measurement to be set as reference is not a reference measurement') self._refMeas[mtype] = (m.UUID, m.name) - with self.file("r+") as f: - # Update attribute in file. Stored as a flat list of string tuples: - # [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...] - reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items()) - # print(reflist) - f.attrs['refMeas'] = reflist + self.__storeReafMeas() @staticmethod def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]): @@ -657,13 +688,13 @@ class Measurement: Nblock = block.shape[0] sum_ += np.sum(block, axis=0) N += Nblock - meansquare += np.sum(block**2, axis=0) / self.N + meansquare += np.sum(block ** 2, axis=0) / self.N avg = sum_ / N # In fact, this is not the complete RMS, as in includes the DC # If p = p_dc + p_osc, then rms(p_osc) = sqrt(ms(p)-ms(p_dc)) if substract_average: - meansquare -= avg**2 + meansquare -= avg ** 2 rms = np.sqrt(meansquare) return rms @@ -768,7 +799,7 @@ class Measurement: signal = self.data(channels) # Estimate noise power in block - blocks = [signal[i * N : (i + 1) * N] for i in range(Nblocks)] + blocks = [signal[i * N: (i + 1) * N] for i in range(Nblocks)] if noiseCorrection: # The difference between the measured signal in the previous block and @@ -972,10 +1003,11 @@ class Measurement: raise ValueError(f"File {fn} does not exist.") if timestamp is None: timestamp = os.path.getmtime(fn) + if mfn is None: - mfn = os.path.splitext(fn)[0] + ".h5" + mfn = os.path.splitext(fn)[0] + DOTMEXT else: - mfn = os.path.splitext(mfn)[0] + ".h5" + mfn = os.path.splitext(mfn)[0] + DOTMEXT dat = np.loadtxt(fn, skiprows=skiprows, delimiter=delimiter) if firstcoltime: From 311a1274bf4d7f02c01421a83de20e9ca80cf0de Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Tue, 19 Dec 2023 14:03:46 +0100 Subject: [PATCH 05/12] Added fromFile() method to overcome problem of multiple times opening same file --- python_src/lasp/lasp_measurement.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index f415a02..17010ee 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -970,6 +970,22 @@ class Measurement: wavfile.write(fn, int(self.samplerate), data.astype(newtype)) + @staticmethod + def fromFile(fn): + """ + Try to open measurement from a given file name. First checks + whether the measurement is already open. Otherwise it might + happen that a Measurement object is created twice for the same backing file, which we do not allow. + """ + # See if the base part of the filename is referring to a file that is already open + with h5.File(fn, 'r') as f: + uuid = f.attrs['UUID'] + + if uuid in Measurement.uuid_s.keys(): + return Measurement.uuid_s[uuid] + + return Measurement(fn) + @staticmethod def fromtxt( fn, From 2cd4c616b3b8718e69ad731c4ececdb8055eabb3 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Tue, 19 Dec 2023 14:04:43 +0100 Subject: [PATCH 06/12] Bump 1.2.0 --- pyproject.toml | 2 +- python_src/lasp/lasp_measurement.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index eba62a7..8126b17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.10" description = "Library for Acoustic Signal Processing" license = { "file" = "LICENSE" } authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }] -version = "1.1.0" +version = "1.2.0" keywords = ["DSP", "DAQ", "Signal processing"] diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index 17010ee..c620807 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- from __future__ import annotations - """! Author: J.A. de Jong - ASCEE From 0be8dd71d995a479c92dd49301df645a3bf36c27 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Tue, 19 Dec 2023 14:34:47 +0100 Subject: [PATCH 07/12] Bugfixes: store UUID attribute early when recording is done. Some small improvements --- python_src/lasp/lasp_measurement.py | 33 +++++++++++++++++------------ python_src/lasp/lasp_record.py | 2 ++ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index c620807..09a2151 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -126,7 +126,7 @@ def scaleBlockSens(block, sens): fac = 2 ** (8 * sw - 1) - 1 else: fac = 1.0 - return block.astype(LASP_NUMPY_FLOAT_TYPE) / fac / sens[np.newaxis, :] + return block.astype(LASP_NUMPY_FLOAT_TYPE) / fac / sens[np.newaxis,:] class IterRawData: @@ -199,7 +199,7 @@ class IterRawData: # print(f'block: {block}, starto: {start_offset}, stopo {stop_offset}') self.i += 1 - return fa[block, start_offset:stop_offset, :][:, self.channels] + return fa[block, start_offset:stop_offset,:][:, self.channels] class IterData(IterRawData): @@ -235,9 +235,6 @@ class Measurement: # Full filepath self.fn = fn - # Folder, Base filename + extension - self.folder, self.fn_base = os.path.split(fn) - # Open the h5 file in read-plus mode, to allow for changing the # measurement comment. with h5.File(fn, "r") as f: @@ -365,15 +362,23 @@ class Measurement: def rename(self, newname: str): """ Try to rename the measurement file. + + Args: + newname: New name, with or without extension """ _ , ext = os.path.splitext(newname) # Add proper extension if new name is given without extension. if ext != DOTMEXT: newname = newname + DOTMEXT - newname_full = str(pathlib.Path(self.folder) / newname) + # Folder, Base filename + extension + folder, _ = os.path.split(self.fn) + + newname_full = str(pathlib.Path(folder) / newname) os.rename(self.fn, newname_full) + self.fn = newname_full + def genNewUUID(self): """ Create new UUID for measurement and store in file. @@ -419,7 +424,8 @@ class Measurement: # Last resort, see if we can find the right measurement in the same folder if m is None: try: - m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name]) + folder, _ = os.path.split(self.fn) + m = Measurement.fromFolderWithUUID(required_uuid, folder, skip=[self.name]) logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement') # Update the measurement file name in the list, such that next time it # can be opened just by its name. @@ -497,13 +503,14 @@ class Measurement: raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.') - def setAttribute(self, attrname, value): + def setAttribute(self, attrname: str, value): """ Set an attribute in the measurement file, and keep a local copy in memory for efficient accessing. Args: - atrname + attrname: name of attribute, a string + value: the value. Should be anything that can be stored as an attribute in HDF5. """ with self.file("r+") as f: # Update comment attribute in the file @@ -535,7 +542,8 @@ class Measurement: @property def name(self): """Returns filename base without extension.""" - return os.path.splitext(self.fn_base)[0] + _, fn = os.path.split(self.fn) + return os.path.splitext(fn)[0] @property def channelNames(self): @@ -1063,8 +1071,8 @@ class Measurement: sensitivity, mfn, timestamp=None, - qtys: List[SIQtys] = None, - channelNames: List[str] = None, + qtys: List[SIQtys]=None, + channelNames: List[str]=None, force=False, ) -> Measurement: """ @@ -1126,7 +1134,6 @@ class Measurement: if len(qtys) != nchannels: raise RuntimeError("Illegal length of qtys list given") - with h5.File(mfn, "w") as hf: hf.attrs["samplerate"] = samplerate hf.attrs["sensitivity"] = sensitivity diff --git a/python_src/lasp/lasp_record.py b/python_src/lasp/lasp_record.py index 2b666cf..07172a4 100644 --- a/python_src/lasp/lasp_record.py +++ b/python_src/lasp/lasp_record.py @@ -9,6 +9,7 @@ import numpy as np from .lasp_atomic import Atomic from .lasp_cpp import InDataHandler, StreamMgr from .lasp_version import LASP_VERSION_MAJOR, LASP_VERSION_MINOR +import uuid @dataclasses.dataclass @@ -139,6 +140,7 @@ class Recording: f.attrs["blocksize"] = blocksize f.attrs["sensitivity"] = [ch.sensitivity for ch in in_ch] f.attrs["channelNames"] = [ch.name for ch in in_ch] + f.attrs["UUID"] = str(uuid.uuid1()) # Add the start delay here, as firstFrames() is called right after the # constructor is called. time.time() returns a floating point From 514ed1aa3250335ece75f0d0617c8206746f31d1 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Wed, 10 Jan 2024 12:26:38 +0100 Subject: [PATCH 08/12] Added physicalOutputQty for daq devices, added possibility to inspect from Python whether device has monitor. Added unit for equation in Qtys. Version bump 1.3.0 --- cpp_src/device/lasp_daq.cpp | 13 ++-------- cpp_src/device/lasp_daqconfig.cpp | 2 ++ cpp_src/device/lasp_deviceinfo.h | 36 ++++++++++++++++++++++++---- cpp_src/device/lasp_rtaudiodaq.cpp | 1 + cpp_src/device/lasp_uldaq.cpp | 4 ++++ cpp_src/pybind11/lasp_deviceinfo.cpp | 10 ++++++-- pyproject.toml | 2 +- python_src/lasp/lasp_common.py | 15 +++++++++++- 8 files changed, 64 insertions(+), 19 deletions(-) diff --git a/cpp_src/device/lasp_daq.cpp b/cpp_src/device/lasp_daq.cpp index 24374a2..7dfbdcd 100644 --- a/cpp_src/device/lasp_daq.cpp +++ b/cpp_src/device/lasp_daq.cpp @@ -44,22 +44,13 @@ Daq::Daq(const DeviceInfo &devinfo, const DaqConfiguration &config) : DaqConfiguration(config), DeviceInfo(devinfo) { DEBUGTRACE_ENTER; - if (duplexMode()) { - if (neninchannels() == 0) { - throw rte("Duplex mode enabled, but no input channels enabled"); - } - - if (nenoutchannels() == 0) { - throw rte("Duplex mode enabled, but no output channels enabled"); - } - } if(!duplexMode() && monitorOutput) { - throw rte("Output monitoring only allowed when running in duplex mode"); + throw rte("Duplex mode requires enabling both input and output channels. Please make sure at least one output channel is enabled, or disable hardware output loopback in DAQ configuration."); } if (!hasInternalOutputMonitor && monitorOutput) { throw rte( - "Output monitor flag set, but device does not have output monitor"); + "Output monitor flag set, but device does not have hardware output monitor."); } if (!config.match(devinfo)) { diff --git a/cpp_src/device/lasp_daqconfig.cpp b/cpp_src/device/lasp_daqconfig.cpp index 3cde4a0..8f503f3 100644 --- a/cpp_src/device/lasp_daqconfig.cpp +++ b/cpp_src/device/lasp_daqconfig.cpp @@ -35,12 +35,14 @@ DaqConfiguration::DaqConfiguration(const DeviceInfo &device) { us i = 0; for (auto &inch : inchannel_config) { inch.name = "Unnamed input channel " + std::to_string(i); + inch.rangeIndex = device.prefInputRangeIndex; i++; } i = 0; for (auto &outch : outchannel_config) { outch.name = "Unnamed output channel " + std::to_string(i); + outch.rangeIndex = device.prefOutputRangeIndex; i++; } diff --git a/cpp_src/device/lasp_deviceinfo.h b/cpp_src/device/lasp_deviceinfo.h index 7d3d8ad..ffd221a 100644 --- a/cpp_src/device/lasp_deviceinfo.h +++ b/cpp_src/device/lasp_deviceinfo.h @@ -68,14 +68,26 @@ public: us prefFramesPerBlockIndex = 0; /** - * @brief Available ranges for the input, i.e. +/- 1V and/or +/- 10 V etc. + * @brief Available ranges for the input, i.e. +/- 1V and/or +/- 10 V etc. */ dvec availableInputRanges; + /** - * @brief Its preffered range + * @brief Available ranges for the output, i.e. +/- 1V and/or +/- 10 V etc. + */ + dvec availableOutputRanges; + + /** + * @brief Its preffered input range */ int prefInputRangeIndex = 0; + /** + * @brief Its preffered output range + */ + int prefOutputRangeIndex = 0; + + /** * @brief The number of input channels available for the device */ @@ -125,13 +137,29 @@ public: bool duplexModeForced = false; /** - * @brief The physical quantity of the output signal. For 'normal' audio + * @brief Indicates whether the device is able to run in duplex mode. If false, + * devices cannot run in duplex mode, and the `duplexModeForced` flag is meaningless. + */ + bool hasDuplexMode = false; + + /** + * @brief The physical quantity of the input signal from DAQ. For 'normal' audio + * interfaces, this is typically a 'number' between +/- full scale. For some + * real DAQ devices however, the input quantity corresponds to a physical signal, + * such a Volts. + */ + + DaqChannel::Qty physicalInputQty = DaqChannel::Qty::Number; + + /** + * @brief The physical quantity of the output signal from DAQ. For 'normal' audio * devices, this is typically a 'number' between +/- full scale. For some - * devices however, the output quantity corresponds to a physical signal, + * real DAQ devices however, the input quantity corresponds to a physical signal, * such a Volts. */ DaqChannel::Qty physicalOutputQty = DaqChannel::Qty::Number; + /** * @brief String representation of DeviceInfo * diff --git a/cpp_src/device/lasp_rtaudiodaq.cpp b/cpp_src/device/lasp_rtaudiodaq.cpp index f797fa0..c6acbb9 100644 --- a/cpp_src/device/lasp_rtaudiodaq.cpp +++ b/cpp_src/device/lasp_rtaudiodaq.cpp @@ -99,6 +99,7 @@ void fillRtAudioDeviceInfo(DeviceInfoList &devinfolist) { d.ninchannels = devinfo.inputChannels; d.availableInputRanges = {1.0}; + d.availableOutputRanges = {1.0}; RtAudioFormat formats = devinfo.nativeFormats; if (formats & RTAUDIO_SINT8) { diff --git a/cpp_src/device/lasp_uldaq.cpp b/cpp_src/device/lasp_uldaq.cpp index 04c21e5..b1f215e 100644 --- a/cpp_src/device/lasp_uldaq.cpp +++ b/cpp_src/device/lasp_uldaq.cpp @@ -68,6 +68,7 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) { } devinfo.physicalOutputQty = DaqChannel::Qty::Voltage; + devinfo.physicalInputQty = DaqChannel::Qty::Voltage; devinfo.availableDataTypes.push_back( DataTypeDescriptor::DataType::dtype_fl64); @@ -79,7 +80,9 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) { devinfo.availableFramesPerBlock = {512, 1024, 2048, 4096, 8192}; devinfo.availableInputRanges = {1.0, 10.0}; + devinfo.availableOutputRanges = {10.0}; devinfo.prefInputRangeIndex = 0; + devinfo.prefOutputRangeIndex = 0; devinfo.ninchannels = 4; devinfo.noutchannels = 1; @@ -90,6 +93,7 @@ void fillUlDaqDeviceInfo(DeviceInfoList &devinfolist) { devinfo.hasInternalOutputMonitor = true; + devinfo.hasDuplexMode = true; devinfo.duplexModeForced = true; // Finally, this devinfo is pushed back in list diff --git a/cpp_src/pybind11/lasp_deviceinfo.cpp b/cpp_src/pybind11/lasp_deviceinfo.cpp index 6034fcc..880f065 100644 --- a/cpp_src/pybind11/lasp_deviceinfo.cpp +++ b/cpp_src/pybind11/lasp_deviceinfo.cpp @@ -29,6 +29,9 @@ void init_deviceinfo(py::module& m) { devinfo.def_readonly("availableInputRanges", &DeviceInfo::availableInputRanges); devinfo.def_readonly("prefInputRangeIndex", &DeviceInfo::prefInputRangeIndex); + devinfo.def_readonly("availableOutputRanges", + &DeviceInfo::availableOutputRanges); + devinfo.def_readonly("prefOutputRangeIndex", &DeviceInfo::prefOutputRangeIndex); devinfo.def_readonly("ninchannels", &DeviceInfo::ninchannels); devinfo.def_readonly("noutchannels", &DeviceInfo::noutchannels); @@ -36,7 +39,10 @@ void init_deviceinfo(py::module& m) { devinfo.def_readonly("hasInputACCouplingSwitch", &DeviceInfo::hasInputACCouplingSwitch); + devinfo.def_readonly("hasDuplexMode", &DeviceInfo::hasDuplexMode); + devinfo.def_readonly("duplexModeForced", &DeviceInfo::duplexModeForced); + devinfo.def_readonly("hasInternalOutputMonitor", &DeviceInfo::hasInternalOutputMonitor); + + devinfo.def_readonly("physicalInputQty", &DeviceInfo::physicalInputQty); devinfo.def_readonly("physicalOutputQty", &DeviceInfo::physicalOutputQty); - } - diff --git a/pyproject.toml b/pyproject.toml index 8126b17..7eca1e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.10" description = "Library for Acoustic Signal Processing" license = { "file" = "LICENSE" } authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }] -version = "1.2.0" +version = "1.3.0" keywords = ["DSP", "DAQ", "Signal processing"] diff --git a/python_src/lasp/lasp_common.py b/python_src/lasp/lasp_common.py index 2594b37..c0c69ba 100644 --- a/python_src/lasp/lasp_common.py +++ b/python_src/lasp/lasp_common.py @@ -62,8 +62,9 @@ class Qty: name: str # I.e.: Pascal unit_name: str - # I.e.: Pa + # I.e.: -, Pa, V unit_symb: str + # I.e.: ('dB SPL') <== tuple of possible level units level_unit: object # Contains a tuple of possible level names, including its reference value. @@ -92,6 +93,18 @@ class Qty: """ return self.cpp_enum.value + @property + def unit_symb_eq(self): + """Unit symbol to be used in equations + + Returns: + String: V, Pa, 1, + """ + if self.unit_symb != '-': + return self.unit_symb + else: + return '1' + @unique From 695a05b262dc685c4ebf842433563d13a995f348 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Wed, 10 Jan 2024 13:01:07 +0100 Subject: [PATCH 09/12] BUGFIX: Prevent corrupting all files when no UUID is yet stored in a file --- python_src/lasp/lasp_measurement.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index 09a2151..47b0093 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -986,7 +986,13 @@ class Measurement: """ # See if the base part of the filename is referring to a file that is already open with h5.File(fn, 'r') as f: - uuid = f.attrs['UUID'] + try: + uuid = f.attrs['UUID'] + except AttributeError: + # No UUID stored in measurement. This is an old measurement that did not have UUID's + # We create a new UUID here such that the file is opened from the filesystem + # anyhow. + uuid = str(uuid.uuid1()) if uuid in Measurement.uuid_s.keys(): return Measurement.uuid_s[uuid] From 14ab3d9dfe4ab56491b74bb3703e3d81837c4d8b Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Wed, 10 Jan 2024 13:01:38 +0100 Subject: [PATCH 10/12] Version bump: bugfix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7eca1e7..78e48af 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.10" description = "Library for Acoustic Signal Processing" license = { "file" = "LICENSE" } authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }] -version = "1.3.0" +version = "1.3.1" keywords = ["DSP", "DAQ", "Signal processing"] From e8ba3b86bfe9b00db71c97a0b19153f646b2ae89 Mon Sep 17 00:00:00 2001 From: Thijs Hekman Date: Wed, 10 Jan 2024 13:14:54 +0100 Subject: [PATCH 11/12] Bugfix on bugfix. KeyError instead of AttributeError --- python_src/lasp/lasp_measurement.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index 47b0093..381466d 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -987,15 +987,15 @@ class Measurement: # See if the base part of the filename is referring to a file that is already open with h5.File(fn, 'r') as f: try: - uuid = f.attrs['UUID'] - except AttributeError: + theuuid = f.attrs['UUID'] + except KeyError: # No UUID stored in measurement. This is an old measurement that did not have UUID's # We create a new UUID here such that the file is opened from the filesystem # anyhow. - uuid = str(uuid.uuid1()) + theuuid = str(uuid.uuid1()) - if uuid in Measurement.uuid_s.keys(): - return Measurement.uuid_s[uuid] + if theuuid in Measurement.uuid_s.keys(): + return Measurement.uuid_s[theuuid] return Measurement(fn) From 061beaf88bdfdb41d75d1166470986867d1fb49a Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Wed, 10 Jan 2024 17:30:56 +0100 Subject: [PATCH 12/12] Small bugfix of some dead code --- python_src/lasp/filter/filterbank_design.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_src/lasp/filter/filterbank_design.py b/python_src/lasp/filter/filterbank_design.py index 7ca0903..dfaccb8 100644 --- a/python_src/lasp/filter/filterbank_design.py +++ b/python_src/lasp/filter/filterbank_design.py @@ -209,6 +209,7 @@ class FilterBankDesigner: Returns: h: Linear filter transfer function [-] """ + fs = self.fs fir = self.createFirFilter(fs, x) # Decimated sampling frequency [Hz]