Merge remote-tracking branch 'origin/develop' into develop

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required (VERSION 3.16) 
-project(LASP LANGUAGES C CXX VERSION 1.0)
+project(LASP LANGUAGES C CXX VERSION 1.1)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED)

pyproject.toml

@@ -5,7 +5,7 @@ requires-python = ">=3.10"
 description = "Library for Acoustic Signal Processing"
 license = { "file" = "LICENSE" }
 authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }]
-version = "1.0.1"
+version = "1.1.0"
 
 keywords = ["DSP", "DAQ", "Signal processing"]
 

python_src/lasp/lasp_measurement.py

@@ -11,18 +11,21 @@ The ASCEE hdf5 measurement file format contains the following fields:
 
 - Attributes:
 
-'version': If not given, version 1 is assumed. For version 1, measurement data
-is assumed to be acoustic data.
-'samplerate': The audio data sample rate in Hz.
-'nchannels': The number of audio channels in the file
+'LASP_VERSION_MAJOR': int The major version of LASP which which the recording has been performed.
+'LASP_VERSION_MINOR': int The minor version of LASP which which the recording has been performed.
+
+'samplerate': The audio data sample rate in Hz [float]
+'nchannels': The number of audio channels in the file List[float]
 'sensitivity': (Optionally) the stored sensitivity of the record channels.
                This can be a single value, or a list of sensitivities for
-               each channel. Both representations are allowed.
+               each channel. Both representations are allowed. List[float]
 
 For measurement files of LASP < v1.0
 'qtys' : (Optionally): list of quantities that is recorded for each channel',
 if this array is not found. Quantities are defaulted to 'Number / Full scale'
 
+'type_int': A specified measurement type that can be used programmatically. It can be read out as an enumeration variant of type "MeasurementType". See code below of implemented measurement types.
+
 For measurement files of LASP >= 1.0
 
 - Datasets:
@@ -32,7 +35,7 @@ block index, the second axis the sample number and the third axis is the
 channel number. The data type is either int16, int32 or float64 / float32. If
 raw data is stored as integer values (int16, int32), the actual values should
 be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the
-corresponding 'number' lies between -1.0 and 1.0.
+corresponding 'number' lies between -1.0 and 1.0. To stay backwards-compatible, the dataset is always called 'audio' despite it being possible that other types of data is stored in the dataset (such as voltages, accelerations etc).
 
 'video': 4-dimensional array of video frames. The first index is the frame
          number, the second the x-value of the pixel and the third is the
@@ -44,10 +47,16 @@ The video dataset can possibly be not present in the data.
 
 """
 
-__all__ = ["Measurement", "scaleBlockSens"]
+__all__ = ["Measurement", "scaleBlockSens", "MeasurementType"]
 from contextlib import contextmanager
+from weakref import WeakValueDictionary
 import h5py as h5
+import uuid
+import pathlib
+import glob
+import itertools
 import numpy as np
+from enum import Enum, unique
 from .lasp_config import LASP_NUMPY_FLOAT_TYPE
 from scipy.io import wavfile
 import os, time, wave, logging
@@ -57,6 +66,32 @@ from .lasp_cpp import Window, DaqChannel, AvPowerSpectra
 from typing import List
 from functools import lru_cache
 
+# Measurement file extension
+MEXT = 'h5'
+DOTMEXT = f'.{MEXT}'
+
+
+@unique
+class MeasurementType(Enum):
+    """
+    Measurement flags related to the measurement. Stored as bit flags in the measurement file. This is for possible changes in the API later.
+    """
+
+    # Not specific measurement type    
+    NotSpecific = 0
+    
+    # Measurement serves as an insertion loss reference measurement
+    ILReference = 1 << 0
+
+    # Measurement is general calibration measurement (to calibrate sensor in a certain way)
+    CALGeneral = 1 << 1
+
+    # Measurement serves as impedance tube calibration (short tube case / ref. plane at origin)
+    muZCalShort = 1 << 2
+
+    # Measurement serves as impedance tube calibration (long tube case)
+    muZCalLong = 1 << 3
+
 
 def getSampWidth(dtype):
     """Returns the width of a single sample in **bytes**.
@@ -188,16 +223,21 @@ class Measurement:
     """Provides access to measurement data stored in the h5 measurement file
     format."""
 
+    # Store a dict of open measurements, with uuid string as a key. We store them as a weak ref.
+    uuid_s = WeakValueDictionary()
+
     def __init__(self, fn):
         """Initialize a Measurement object based on the filename."""
-        if ".h5" not in fn:
-            fn += ".h5"
+
+        # Add extension if tried to open without exension
+        if DOTMEXT not in fn:
+            fn += DOTMEXT
 
         # Full filepath
         self.fn = fn
 
-        # Base filename
-        self.fn_base = os.path.split(fn)[1]
+        # Folder, Base filename + extension
+        self.folder, self.fn_base = os.path.split(fn)
 
         # Open the h5 file in read-plus mode, to allow for changing the
         # measurement comment.
@@ -222,9 +262,39 @@ class Measurement:
                 self.version_major = f.attrs["LASP_VERSION_MAJOR"]
                 self.version_minor = f.attrs["LASP_VERSION_MINOR"]
             except KeyError:
+                # No version information stored 
                 self.version_major = 0
                 self.version_minor = 1
 
+            try:
+                # Try to catch UUID (Universally Unique IDentifier)
+                self._UUID = f.attrs['UUID']
+                # Flag indicating we have to add a new UUID
+                create_new_uuid = False
+            except KeyError:
+                create_new_uuid = True
+
+            try:
+                # UUID of the reference measurement. Should be stored as 
+                # a lists of tuples, where each tuple is a combination of (<MeasurementType.value>, <uuid_string>, <last_filename>).
+
+                # The last filename is a filename that *probably* is the reference measurement with
+                # given UUID. If it is not, we will search for it in the same directory as `this` measurement.
+                # If we cannot find it there, we will give up, and remove the field corresponding to this reference measurement type.
+                refMeas_list = f.attrs['refMeas']
+
+                # Build a tuple string from it
+                self._refMeas = {}
+                for (key, val, name) in refMeas_list:
+                    self._refMeas[MeasurementType(int(key))] = (val, name)
+            except KeyError:
+                self._refMeas = {}
+
+            try:
+                self._type_int = f.attrs['type_int']
+            except KeyError:
+                self._type_int = 0
+
             # Due to a previous bug, the channel names were not stored
             # consistently, i.e. as 'channel_names' and later camelcase.
             try:
@@ -282,15 +352,186 @@ class Measurement:
                     f"Physical quantity data not available in measurement file. Assuming {SIQtys.default}"
                 )
 
-    def setAttribute(self, atrname, value):
+        if create_new_uuid:
+            # Create and store a random UUID based on *now* and store it forever
+            # inside of the Measurement file
+            self.genNewUUID()
+        else:
+            if self.UUID in Measurement.uuid_s.keys():
+                raise RuntimeError(f"Measurement '{self.name}' is already opened. Cannot open measurement twice. Note: this error can happen when measurements are manually copied.")
+
+        # Store weak reference to 'self' in list of UUID's. They are removed when no file is open anymore
+        Measurement.uuid_s[self._UUID] = self
+
+    def rename(self, newname: str):
+        """
+        Try to rename the measurement file.
+        """
+        _ , ext = os.path.splitext(newname)
+        # Add proper extension if new name is given without extension.
+        if ext != DOTMEXT:
+            newname = newname + DOTMEXT
+            
+        newname_full = str(pathlib.Path(self.folder) / newname)
+        os.rename(self.fn, newname_full)
+
+    def genNewUUID(self):
+        """
+        Create new UUID for measurement and store in file.
+        """
+        self.setAttribute('UUID', str(uuid.uuid1()))
+
+    @property
+    def UUID(self):
+        """
+        Universally unique identifier
+        """
+        return self._UUID
+    
+    def getRefMeas(self, mtype: MeasurementType):
+        """
+        Return corresponding reference measurement, if configured and can be found. If the reference 
+        measurement is currently not open, it tries to open it by traversing other measurement
+        files in the current directory. Throws a runtime error in case the reference measurement cannot be found.
+        Throws a ValueError when the reference measurement is not configured.
+        """
+        
+        # See if we can find the UUID for the required measurement type
+        try:
+            required_uuid, possible_name = self._refMeas[mtype]
+        except KeyError:
+            raise ValueError(f"No reference measurement configured for '{self.name}'")
+            
+        m = None
+        # Try to find it in the dictionary of of open measurements
+        if required_uuid in Measurement.uuid_s.keys():
+            m = Measurement.uuid_s[required_uuid]
+            logging.info(f'Returned reference measurement {m.name} from list of open measurements')
+        
+        # Not found in list of openend measurements. See if we can open it using its last stored file name we know of
+        if m is None:
+            try:
+                m = Measurement(possible_name)
+                if m.UUID == required_uuid:
+                    logging.info(f'Opened reference measurement {m.name} by name')
+            except Exception as e:
+                logging.error(f'Could not find reference measurement using file name: {possible_name}')
+
+        # Last resort, see if we can find the right measurement in the same folder
+        if m is None:
+            try:
+                m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name])
+                logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement')
+                # Update the measurement file name in the list, such that next time it
+                # can be opened just by its name.
+                self.setRefMeas(m)
+            except:
+                logging.error("Could not find the reference measurement. Is it deleted?")
+
+        # Well, we found it. Now make sure the reference measurement actually has the right type (User could have marked it as a NotSpecific for example in the mean time).
+        if m is not None:
+            if m.measurementType() != mtype:
+                m.removeRefMeas(mtype)
+                raise RuntimeError(f"Reference measurement for {self.name} is not a proper reference (anymore).")
+            
+            # Whow, we passed all security checks, here we go!
+            return m
+        else:
+            # Nope, not there.
+            raise RuntimeError(f"Could not find the reference measurement for '{self.name}'. Is it deleted?")
+
+    def removeRefMeas(self, mtype: MeasurementType):
+        """
+        Remove an existing reference measurement of specified type from this measurement. Silently ignores this
+        action if no reference measurement of this type is configured.
+
+        """
+        try:
+            del self._refMeas[mtype]
+            self.__storeReafMeas()
+        except KeyError:
+            pass
+
+    def __storeReafMeas(self):
+        """
+        Internal method that syncs the dictionary of reference methods with the backing HDF5 file
+        """
+        with self.file("r+") as f:
+            # Update attribute in file. Stored as a flat list of string tuples:
+            # [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...]
+            reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items())
+            # print(reflist)
+            f.attrs['refMeas'] = reflist
+
+    def setRefMeas(self, m: Measurement):
+        """
+        Set a reference measurement for the given measurement. If this measurement is already
+        a reference measurement, the previous reference measurement type is overwritten, such that there is
+        only one measurement that is the reference of a certain 'MeasurementType'
+        """
+        mtype = m.measurementType()
+        if mtype == MeasurementType.NotSpecific:
+            raise ValueError('Measurement to be set as reference is not a reference measurement')
+    
+        self._refMeas[mtype] = (m.UUID, m.name)
+        self.__storeReafMeas()
+
+    @staticmethod
+    def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]):
+        """
+        Returns Measurement object from a given UUID string. It first tries to find whether there
+        is an uuid in the static list of weak references. If not, it will try to open files in 
+        the current file path.
+        """
+        for fn in glob.glob(str(pathlib.Path(folder)) + f'/*{DOTMEXT}'):
+            # Do not try to open this file in case it is in the 'skip' list.
+            if len(list(filter(lambda a: a in fn, skip))) > 0:
+                continue
+                
+            try:
+                m = Measurement(fn)
+                if m.UUID == uuid_str: 
+                    # Update 'last_fn' attribute in dict of stored reference measurements
+                    return m
+            except Exception as e:
+                logging.error(f'Possible measurement file {fn} returned error {e} when opening.')
+
+        raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.')
+
+    def setAttribute(self, attrname, value):
         """
         Set an attribute in the measurement file, and keep a local copy in
         memory for efficient accessing.
+
+        Args:
+            atrname
         """
         with self.file("r+") as f:
             # Update comment attribute in the file
-            f.attrs[atrname] = value
-        setattr(self, "_" + atrname, value)
+            f.attrs[attrname] = value
+        setattr(self, "_" + attrname, value)
+
+    def isType(self, type_: MeasurementType):
+        """
+        Returns True when a measurement is flagged as being of a certaint "MeasurementType"
+        """
+        if (type_.value & self._type_int):
+            return True
+        elif type_.value == self._type_int == 0:
+            return True
+        return False
+    
+    def setType(self, type_: MeasurementType):
+        """
+        Set the measurement type to given type
+        """
+        self.setAttribute('type_int', type_.value)
+
+    def measurementType(self):
+        """
+        Returns type of measurement
+        """
+        return MeasurementType(self._type_int)
 
     @property
     def name(self):
@@ -303,12 +544,18 @@ class Measurement:
 
     @channelNames.setter
     def channelNames(self, newchnames):
+        """
+        Returns list of the names of the channels
+        """
         if len(newchnames) != self.nchannels:
             raise RuntimeError("Invalid length of new channel names")
         self.setAttribute("channelNames", newchnames)
 
     @property
     def channelConfig(self):
+        """
+        Returns list of current channel configuration data.
+        """
         chcfg = []
         for chname, sens, qty in zip(self.channelNames, self.sensitivity, self.qtys):
             ch = DaqChannel()
@@ -321,6 +568,18 @@ class Measurement:
 
     @channelConfig.setter
     def channelConfig(self, chcfg: List[DaqChannel]):
+        """
+        Set new channel configuration from list of DaqChannel objects. 
+
+        Use cases:
+            - Update channel types, sensitivities etc.
+
+        Args:
+            chchfg: New channel configuration
+        """
+        if len(chcfg) != self.nchannels:
+            raise RuntimeError("Invalid number of channels")
+
         chname = []
         sens = []
         qtys = []
@@ -429,13 +688,13 @@ class Measurement:
                 Nblock = block.shape[0]
                 sum_ += np.sum(block, axis=0)
                 N += Nblock
-                meansquare += np.sum(block**2, axis=0) / self.N
+                meansquare += np.sum(block ** 2, axis=0) / self.N
 
         avg = sum_ / N
         # In fact, this is not the complete RMS, as in includes the DC
         # If p = p_dc + p_osc, then rms(p_osc) = sqrt(ms(p)-ms(p_dc))
         if substract_average:
-            meansquare -= avg**2
+            meansquare -= avg ** 2
         rms = np.sqrt(meansquare)
         return rms
 
@@ -540,7 +799,7 @@ class Measurement:
         signal = self.data(channels)
 
         # Estimate noise power in block
-        blocks = [signal[i * N : (i + 1) * N] for i in range(Nblocks)]
+        blocks = [signal[i * N: (i + 1) * N] for i in range(Nblocks)]
 
         if noiseCorrection:
             # The difference between the measured signal in the previous block and
@@ -744,10 +1003,11 @@ class Measurement:
             raise ValueError(f"File {fn} does not exist.")
         if timestamp is None:
             timestamp = os.path.getmtime(fn)
+
         if mfn is None:
-            mfn = os.path.splitext(fn)[0] + ".h5"
+            mfn = os.path.splitext(fn)[0] + DOTMEXT
         else:
-            mfn = os.path.splitext(mfn)[0] + ".h5"
+            mfn = os.path.splitext(mfn)[0] + DOTMEXT
 
         dat = np.loadtxt(fn, skiprows=skiprows, delimiter=delimiter)
         if firstcoltime:
@@ -818,8 +1078,9 @@ class Measurement:
             force: If True, overwrites existing files with specified `mfn`
             name.
         """
-        if os.path.splitext(mfn)[1] != ".h5":
-            mfn += ".h5"
+
+        if os.path.splitext(mfn)[1] != DOTMEXT:
+            mfn += DOTMEXT
         if os.path.exists(mfn) and not force:
             raise ValueError(f"File {mfn} already exist.")
         if timestamp is None:
@@ -915,3 +1176,4 @@ class Measurement:
             ad[0] = data
 
         return Measurement(newfn)
+