From c610c6350d4a0b9bde5de45b0829b7a331d03060 Mon Sep 17 00:00:00 2001 From: "J.A. de Jong - Redu-Sone B.V., ASCEE V.O.F" Date: Mon, 18 Dec 2023 12:37:46 +0100 Subject: [PATCH] Implemented reference measurements, and renaming as method in Measurement object. --- CMakeLists.txt | 2 +- pyproject.toml | 2 +- python_src/lasp/lasp_measurement.py | 262 ++++++++++++++++++++++++++-- 3 files changed, 248 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43173bf..d28253b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required (VERSION 3.16) -project(LASP LANGUAGES C CXX VERSION 1.0) +project(LASP LANGUAGES C CXX VERSION 1.1) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED) diff --git a/pyproject.toml b/pyproject.toml index a2aface..eba62a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires-python = ">=3.10" description = "Library for Acoustic Signal Processing" license = { "file" = "LICENSE" } authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }] -version = "1.0.1" +version = "1.1.0" keywords = ["DSP", "DAQ", "Signal processing"] diff --git a/python_src/lasp/lasp_measurement.py b/python_src/lasp/lasp_measurement.py index f798577..b2e9d85 100644 --- a/python_src/lasp/lasp_measurement.py +++ b/python_src/lasp/lasp_measurement.py @@ -11,18 +11,21 @@ The ASCEE hdf5 measurement file format contains the following fields: - Attributes: -'version': If not given, version 1 is assumed. For version 1, measurement data -is assumed to be acoustic data. -'samplerate': The audio data sample rate in Hz. -'nchannels': The number of audio channels in the file +'LASP_VERSION_MAJOR': int The major version of LASP which which the recording has been performed. +'LASP_VERSION_MINOR': int The minor version of LASP which which the recording has been performed. + +'samplerate': The audio data sample rate in Hz [float] +'nchannels': The number of audio channels in the file List[float] 'sensitivity': (Optionally) the stored sensitivity of the record channels. This can be a single value, or a list of sensitivities for - each channel. Both representations are allowed. + each channel. Both representations are allowed. List[float] For measurement files of LASP < v1.0 'qtys' : (Optionally): list of quantities that is recorded for each channel', if this array is not found. Quantities are defaulted to 'Number / Full scale' +'type_int': A specified measurement type that can be used programmatically. It can be read out as an enumeration variant of type "MeasurementType". See code below of implemented measurement types. + For measurement files of LASP >= 1.0 - Datasets: @@ -32,7 +35,7 @@ block index, the second axis the sample number and the third axis is the channel number. The data type is either int16, int32 or float64 / float32. If raw data is stored as integer values (int16, int32), the actual values should be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the -corresponding 'number' lies between -1.0 and 1.0. +corresponding 'number' lies between -1.0 and 1.0. To stay backwards-compatible, the dataset is always called 'audio' despite it being possible that other types of data is stored in the dataset (such as voltages, accelerations etc). 'video': 4-dimensional array of video frames. The first index is the frame number, the second the x-value of the pixel and the third is the @@ -44,10 +47,16 @@ The video dataset can possibly be not present in the data. """ -__all__ = ["Measurement", "scaleBlockSens"] +__all__ = ["Measurement", "scaleBlockSens", "MeasurementType"] from contextlib import contextmanager +from weakref import WeakValueDictionary import h5py as h5 +import uuid +import pathlib +import glob +import itertools import numpy as np +from enum import Enum, unique from .lasp_config import LASP_NUMPY_FLOAT_TYPE from scipy.io import wavfile import os, time, wave, logging @@ -57,6 +66,32 @@ from .lasp_cpp import Window, DaqChannel, AvPowerSpectra from typing import List from functools import lru_cache +# Measurement file extension +MEXT = 'h5' +DOTMEXT = f'.{MEXT}' + + +@unique +class MeasurementType(Enum): + """ + Measurement flags related to the measurement. Stored as bit flags in the measurement file. This is for possible changes in the API later. + """ + + # Not specific measurement type + NotSpecific = 0 + + # Measurement serves as an insertion loss reference measurement + ILReference = 1 << 0 + + # Measurement is general calibration measurement (to calibrate sensor in a certain way) + CALGeneral = 1 << 1 + + # Measurement serves as impedance tube calibration (short tube case / ref. plane at origin) + muZCalShort = 1 << 2 + + # Measurement serves as impedance tube calibration (long tube case) + muZCalLong = 1 << 3 + def getSampWidth(dtype): """Returns the width of a single sample in **bytes**. @@ -188,16 +223,21 @@ class Measurement: """Provides access to measurement data stored in the h5 measurement file format.""" + # Store a dict of open measurements, with uuid string as a key. We store them as a weak ref. + uuid_s = WeakValueDictionary() + def __init__(self, fn): """Initialize a Measurement object based on the filename.""" - if ".h5" not in fn: - fn += ".h5" + + # Add extension if tried to open without exension + if DOTMEXT not in fn: + fn += DOTMEXT # Full filepath self.fn = fn - # Base filename - self.fn_base = os.path.split(fn)[1] + # Folder, Base filename + extension + self.folder, self.fn_base = os.path.split(fn) # Open the h5 file in read-plus mode, to allow for changing the # measurement comment. @@ -222,9 +262,39 @@ class Measurement: self.version_major = f.attrs["LASP_VERSION_MAJOR"] self.version_minor = f.attrs["LASP_VERSION_MINOR"] except KeyError: + # No version information stored self.version_major = 0 self.version_minor = 1 + try: + # Try to catch UUID (Universally Unique IDentifier) + self._UUID = f.attrs['UUID'] + # Flag indicating we have to add a new UUID + create_new_uuid = False + except KeyError: + create_new_uuid = True + + try: + # UUID of the reference measurement. Should be stored as + # a lists of tuples, where each tuple is a combination of (, , ). + + # The last filename is a filename that *probably* is the reference measurement with + # given UUID. If it is not, we will search for it in the same directory as `this` measurement. + # If we cannot find it there, we will give up, and remove the field corresponding to this reference measurement type. + refMeas_list = f.attrs['refMeas'] + + # Build a tuple string from it + self._refMeas = {} + for (key, val, name) in refMeas_list: + self._refMeas[MeasurementType(int(key))] = (val, name) + except KeyError: + self._refMeas = {} + + try: + self._type_int = f.attrs['type_int'] + except KeyError: + self._type_int = 0 + # Due to a previous bug, the channel names were not stored # consistently, i.e. as 'channel_names' and later camelcase. try: @@ -282,15 +352,155 @@ class Measurement: f"Physical quantity data not available in measurement file. Assuming {SIQtys.default}" ) - def setAttribute(self, atrname, value): + if create_new_uuid: + # Create and store a random UUID based on *now* and store it forever + # inside of the Measurement file + self.genNewUUID() + else: + if self.UUID in Measurement.uuid_s.keys(): + raise RuntimeError(f"Measurement '{self.name}' is already opened. Cannot open measurement twice. Note: this error can happen when measurements are manually copied.") + + # Store weak reference to 'self' in list of UUID's. They are removed when no file is open anymore + Measurement.uuid_s[self._UUID] = self + + def rename(self, newname: str): + """ + Try to rename the measurement file. + """ + _ , ext = os.path.splitext(newname) + # Add proper extension if new name is given without extension. + if ext != DOTMEXT: + newname = newname + DOTMEXT + + newname_full = str(pathlib.Path(self.folder) / newname ) + os.rename(self.fn, newname_full) + + def genNewUUID(self): + """ + Create new UUID for measurement and store in file. + """ + self.setAttribute('UUID', str(uuid.uuid4())) + + @property + def UUID(self): + """ + Universally unique identifier + """ + return self._UUID + + def refMeas(self, mtype: MeasurementType): + """ + Return corresponding reference measurement, if configured and can be found. If the reference + measurement is currently not open, it tries to open it by traversing other measurement + files in the current directory. Throws a runtime error in case the reference measurement cannot be found. + Throws a ValueError when the reference measurement is not configured. + """ + + # See if we can find the UUID for the required measurement type + try: + required_uuid, possible_name = self._refMeas[mtype] + except KeyError: + raise ValueError(f"No reference measurement configured for '{self.name}'") + + # Try to find it in the dictionary of of open measurements + if required_uuid in Measurement.uuid_s.keys(): + m = Measurement.uuid_s[required_uuid] + logging.info(f'Returned reference measurement {m.name} from list of open measurements') + return m + + # See if we can open it using its last stored file name + try: + m = Measurement(possible_name) + if m.UUID == required_uuid: + logging.info(f'Opened reference measurement {m.name} by name') + return m + except Exception as e: + logging.error(f'Could not find reference measurement using file name: {possible_name}') + + # Last resort, see if we can find the measurement in the same folder + m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name]) + logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement') + # Update the measurement file name in the list, such that next time it + # can be opened just by its name. + self.setRefMeas(m) + return m + + raise RuntimeError("Could not find reference measurement") + + def setRefMeas(self, m: Measurement): + """ + Set a reference measurement for the given measurement. If this measurement is already + a reference measurement, the previous reference measurement type is overwritten, such that there is + only one measurement that is the reference of a certain 'MeasurementType' + """ + mtype = m.measurementType() + if mtype == MeasurementType.NotSpecific: + raise ValueError('Measurement to be set as reference is not a reference measurement') + + self._refMeas[mtype] = (m.UUID, m.name) + with self.file("r+") as f: + # Update attribute in file. Stored as a flat list of string tuples: + # [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...] + reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items()) + # print(reflist) + f.attrs['refMeas'] = reflist + + @staticmethod + def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]): + """ + Returns Measurement object from a given UUID string. It first tries to find whether there + is an uuid in the static list of weak references. If not, it will try to open files in + the current file path. + """ + for fn in glob.glob(str(pathlib.Path(folder)) + f'/*{DOTMEXT}'): + # Do not try to open this file in case it is in the 'skip' list. + if len(list(filter(lambda a: a in fn, skip))) > 0: + continue + + try: + m = Measurement(fn) + if m.UUID == uuid_str: + # Update 'last_fn' attribute in dict of stored reference measurements + return m + except Exception as e: + logging.error(f'Possible measurement file {fn} returned error {e} when opening.') + + raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.') + + def setAttribute(self, attrname, value): """ Set an attribute in the measurement file, and keep a local copy in memory for efficient accessing. + + Args: + atrname """ with self.file("r+") as f: # Update comment attribute in the file - f.attrs[atrname] = value - setattr(self, "_" + atrname, value) + f.attrs[attrname] = value + setattr(self, "_" + attrname, value) + + def isType(self, type_: MeasurementType): + """ + Returns True when a measurement is flagged as being of a certaint "MeasurementType" + """ + if (type_.value & self._type_int): + return True + elif type_.value == self._type_int == 0: + return True + return False + + def setType(self, type_: MeasurementType): + """ + Set the measurement type to given type + """ + self.setAttribute('type_int', type_.value) + + def measurementType(self): + """ + Returns type of measurement + """ + return MeasurementType(self._type_int) @property def name(self): @@ -303,12 +513,18 @@ class Measurement: @channelNames.setter def channelNames(self, newchnames): + """ + Returns list of the names of the channels + """ if len(newchnames) != self.nchannels: raise RuntimeError("Invalid length of new channel names") self.setAttribute("channelNames", newchnames) @property def channelConfig(self): + """ + Returns list of current channel configuration data. + """ chcfg = [] for chname, sens, qty in zip(self.channelNames, self.sensitivity, self.qtys): ch = DaqChannel() @@ -321,6 +537,18 @@ class Measurement: @channelConfig.setter def channelConfig(self, chcfg: List[DaqChannel]): + """ + Set new channel configuration from list of DaqChannel objects. + + Use cases: + - Update channel types, sensitivities etc. + + Args: + chchfg: New channel configuration + """ + if len(chcfg) != self.nchannels: + raise RuntimeError("Invalid number of channels") + chname = [] sens = [] qtys = [] @@ -818,8 +1046,9 @@ class Measurement: force: If True, overwrites existing files with specified `mfn` name. """ - if os.path.splitext(mfn)[1] != ".h5": - mfn += ".h5" + + if os.path.splitext(mfn)[1] != DOTMEXT: + mfn += DOTMEXT if os.path.exists(mfn) and not force: raise ValueError(f"File {mfn} already exist.") if timestamp is None: @@ -916,3 +1145,4 @@ class Measurement: ad[0] = data return Measurement(newfn) +