ASCEE/lasp
ASCEE/lasp
5
1
Fork 0
Code Issues 5 Pull Requests Actions Releases 14 Activity

Implemented reference measurements, and renaming as method in Measurement object.

Browse Source
This commit is contained in:
Anne de Jong 2023-12-18 12:37:46 +01:00
parent a7b219a1e1
commit c610c6350d
3 changed files with 248 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -1,5 +1,5 @@
cmake_minimum_required (VERSION 3.16)
project(LASP LANGUAGES C CXX VERSION 1.0)
project(LASP LANGUAGES C CXX VERSION 1.1)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED)

2
pyproject.toml
View File

@ -5,7 +5,7 @@ requires-python = ">=3.10"
description = "Library for Acoustic Signal Processing"
license = { "file" = "LICENSE" }
authors = [{ "name" = "J.A. de Jong", "email" = "j.a.dejong@ascee.nl" }]
version = "1.0.1"
version = "1.1.0"
keywords = ["DSP", "DAQ", "Signal processing"]

262
python_src/lasp/lasp_measurement.py
View File

@ -11,18 +11,21 @@ The ASCEE hdf5 measurement file format contains the following fields:
- Attributes:
'version': If not given, version 1 is assumed. For version 1, measurement data
is assumed to be acoustic data.
'samplerate': The audio data sample rate in Hz.
'nchannels': The number of audio channels in the file
'LASP_VERSION_MAJOR': int The major version of LASP which which the recording has been performed.
'LASP_VERSION_MINOR': int The minor version of LASP which which the recording has been performed.
'samplerate': The audio data sample rate in Hz [float]
'nchannels': The number of audio channels in the file List[float]
'sensitivity': (Optionally) the stored sensitivity of the record channels.
This can be a single value, or a list of sensitivities for
each channel. Both representations are allowed.
each channel. Both representations are allowed. List[float]
For measurement files of LASP < v1.0
'qtys' : (Optionally): list of quantities that is recorded for each channel',
if this array is not found. Quantities are defaulted to 'Number / Full scale'
'type_int': A specified measurement type that can be used programmatically. It can be read out as an enumeration variant of type "MeasurementType". See code below of implemented measurement types.
For measurement files of LASP >= 1.0
- Datasets:
@ -32,7 +35,7 @@ block index, the second axis the sample number and the third axis is the
channel number. The data type is either int16, int32 or float64 / float32. If
raw data is stored as integer values (int16, int32), the actual values should
be pre-scaled by its maximum positive number (2**(nb-1) - 1), such that the
corresponding 'number' lies between -1.0 and 1.0.
corresponding 'number' lies between -1.0 and 1.0. To stay backwards-compatible, the dataset is always called 'audio' despite it being possible that other types of data is stored in the dataset (such as voltages, accelerations etc).
'video': 4-dimensional array of video frames. The first index is the frame
number, the second the x-value of the pixel and the third is the
@ -44,10 +47,16 @@ The video dataset can possibly be not present in the data.
"""
__all__ = ["Measurement", "scaleBlockSens"]
__all__ = ["Measurement", "scaleBlockSens", "MeasurementType"]
from contextlib import contextmanager
from weakref import WeakValueDictionary
import h5py as h5
import uuid
import pathlib
import glob
import itertools
import numpy as np
from enum import Enum, unique
from .lasp_config import LASP_NUMPY_FLOAT_TYPE
from scipy.io import wavfile
import os, time, wave, logging
@ -57,6 +66,32 @@ from .lasp_cpp import Window, DaqChannel, AvPowerSpectra
from typing import List
from functools import lru_cache
# Measurement file extension
MEXT = 'h5'
DOTMEXT = f'.{MEXT}'
@unique
class MeasurementType(Enum):
"""
Measurement flags related to the measurement. Stored as bit flags in the measurement file. This is for possible changes in the API later.
"""
# Not specific measurement type
NotSpecific = 0
# Measurement serves as an insertion loss reference measurement
ILReference = 1 << 0
# Measurement is general calibration measurement (to calibrate sensor in a certain way)
CALGeneral = 1 << 1
# Measurement serves as impedance tube calibration (short tube case / ref. plane at origin)
muZCalShort = 1 << 2
# Measurement serves as impedance tube calibration (long tube case)
muZCalLong = 1 << 3
def getSampWidth(dtype):
"""Returns the width of a single sample in **bytes**.
@ -188,16 +223,21 @@ class Measurement:
"""Provides access to measurement data stored in the h5 measurement file
format."""
# Store a dict of open measurements, with uuid string as a key. We store them as a weak ref.
uuid_s = WeakValueDictionary()
def __init__(self, fn):
"""Initialize a Measurement object based on the filename."""
if ".h5" not in fn:
fn += ".h5"
# Add extension if tried to open without exension
if DOTMEXT not in fn:
fn += DOTMEXT
# Full filepath
self.fn = fn
# Base filename
self.fn_base = os.path.split(fn)[1]
# Folder, Base filename + extension
self.folder, self.fn_base = os.path.split(fn)
# Open the h5 file in read-plus mode, to allow for changing the
# measurement comment.
@ -222,9 +262,39 @@ class Measurement:
self.version_major = f.attrs["LASP_VERSION_MAJOR"]
self.version_minor = f.attrs["LASP_VERSION_MINOR"]
except KeyError:
# No version information stored
self.version_major = 0
self.version_minor = 1
try:
# Try to catch UUID (Universally Unique IDentifier)
self._UUID = f.attrs['UUID']
# Flag indicating we have to add a new UUID
create_new_uuid = False
except KeyError:
create_new_uuid = True
try:
# UUID of the reference measurement. Should be stored as
# a lists of tuples, where each tuple is a combination of (<MeasurementType.value>, <uuid_string>, <last_filename>).
# The last filename is a filename that *probably* is the reference measurement with
# given UUID. If it is not, we will search for it in the same directory as `this` measurement.
# If we cannot find it there, we will give up, and remove the field corresponding to this reference measurement type.
refMeas_list = f.attrs['refMeas']
# Build a tuple string from it
self._refMeas = {}
for (key, val, name) in refMeas_list:
self._refMeas[MeasurementType(int(key))] = (val, name)
except KeyError:
self._refMeas = {}
try:
self._type_int = f.attrs['type_int']
except KeyError:
self._type_int = 0
# Due to a previous bug, the channel names were not stored
# consistently, i.e. as 'channel_names' and later camelcase.
try:
@ -282,15 +352,155 @@ class Measurement:
f"Physical quantity data not available in measurement file. Assuming {SIQtys.default}"
)
def setAttribute(self, atrname, value):
if create_new_uuid:
# Create and store a random UUID based on *now* and store it forever
# inside of the Measurement file
self.genNewUUID()
else:
if self.UUID in Measurement.uuid_s.keys():
raise RuntimeError(f"Measurement '{self.name}' is already opened. Cannot open measurement twice. Note: this error can happen when measurements are manually copied.")
# Store weak reference to 'self' in list of UUID's. They are removed when no file is open anymore
Measurement.uuid_s[self._UUID] = self
def rename(self, newname: str):
"""
Try to rename the measurement file.
"""
_ , ext = os.path.splitext(newname)
# Add proper extension if new name is given without extension.
if ext != DOTMEXT:
newname = newname + DOTMEXT
newname_full = str(pathlib.Path(self.folder) / newname )
os.rename(self.fn, newname_full)
def genNewUUID(self):
"""
Create new UUID for measurement and store in file.
"""
self.setAttribute('UUID', str(uuid.uuid4()))
@property
def UUID(self):
"""
Universally unique identifier
"""
return self._UUID
def refMeas(self, mtype: MeasurementType):
"""
Return corresponding reference measurement, if configured and can be found. If the reference
measurement is currently not open, it tries to open it by traversing other measurement
files in the current directory. Throws a runtime error in case the reference measurement cannot be found.
Throws a ValueError when the reference measurement is not configured.
"""
# See if we can find the UUID for the required measurement type
try:
required_uuid, possible_name = self._refMeas[mtype]
except KeyError:
raise ValueError(f"No reference measurement configured for '{self.name}'")
# Try to find it in the dictionary of of open measurements
if required_uuid in Measurement.uuid_s.keys():
m = Measurement.uuid_s[required_uuid]
logging.info(f'Returned reference measurement {m.name} from list of open measurements')
return m
# See if we can open it using its last stored file name
try:
m = Measurement(possible_name)
if m.UUID == required_uuid:
logging.info(f'Opened reference measurement {m.name} by name')
return m
except Exception as e:
logging.error(f'Could not find reference measurement using file name: {possible_name}')
# Last resort, see if we can find the measurement in the same folder
m = Measurement.fromFolderWithUUID(required_uuid, self.folder, skip=[self.name])
logging.info('Found reference measurement in folder with correct UUID. Updating name of reference measurement')
# Update the measurement file name in the list, such that next time it
# can be opened just by its name.
self.setRefMeas(m)
return m
raise RuntimeError("Could not find reference measurement")
def setRefMeas(self, m: Measurement):
"""
Set a reference measurement for the given measurement. If this measurement is already
a reference measurement, the previous reference measurement type is overwritten, such that there is
only one measurement that is the reference of a certain 'MeasurementType'
"""
mtype = m.measurementType()
if mtype == MeasurementType.NotSpecific:
raise ValueError('Measurement to be set as reference is not a reference measurement')
self._refMeas[mtype] = (m.UUID, m.name)
with self.file("r+") as f:
# Update attribute in file. Stored as a flat list of string tuples:
# [(ref_value1, uuid_1, name_1), (ref_value2, uuid_2, name_2), ...]
reflist = list((str(key.value), val1, val2) for key, (val1, val2) in self._refMeas.items())
# print(reflist)
f.attrs['refMeas'] = reflist
@staticmethod
def fromFolderWithUUID(uuid_str: str, folder: str='', skip=[]):
"""
Returns Measurement object from a given UUID string. It first tries to find whether there
is an uuid in the static list of weak references. If not, it will try to open files in
the current file path.
"""
for fn in glob.glob(str(pathlib.Path(folder)) + f'/*{DOTMEXT}'):
# Do not try to open this file in case it is in the 'skip' list.
if len(list(filter(lambda a: a in fn, skip))) > 0:
continue
try:
m = Measurement(fn)
if m.UUID == uuid_str:
# Update 'last_fn' attribute in dict of stored reference measurements
return m
except Exception as e:
logging.error(f'Possible measurement file {fn} returned error {e} when opening.')
raise RuntimeError(f'Measurement with UUID {uuid_str} could not be found.')
def setAttribute(self, attrname, value):
"""
Set an attribute in the measurement file, and keep a local copy in
memory for efficient accessing.
Args:
atrname
"""
with self.file("r+") as f:
# Update comment attribute in the file
f.attrs[atrname] = value
setattr(self, "_" + atrname, value)
f.attrs[attrname] = value
setattr(self, "_" + attrname, value)
def isType(self, type_: MeasurementType):
"""
Returns True when a measurement is flagged as being of a certaint "MeasurementType"
"""
if (type_.value & self._type_int):
return True
elif type_.value == self._type_int == 0:
return True
return False
def setType(self, type_: MeasurementType):
"""
Set the measurement type to given type
"""
self.setAttribute('type_int', type_.value)
def measurementType(self):
"""
Returns type of measurement
"""
return MeasurementType(self._type_int)
@property
def name(self):
@ -303,12 +513,18 @@ class Measurement:
@channelNames.setter
def channelNames(self, newchnames):
"""
Returns list of the names of the channels
"""
if len(newchnames) != self.nchannels:
raise RuntimeError("Invalid length of new channel names")
self.setAttribute("channelNames", newchnames)
@property
def channelConfig(self):
"""
Returns list of current channel configuration data.
"""
chcfg = []
for chname, sens, qty in zip(self.channelNames, self.sensitivity, self.qtys):
ch = DaqChannel()
@ -321,6 +537,18 @@ class Measurement:
@channelConfig.setter
def channelConfig(self, chcfg: List[DaqChannel]):
"""
Set new channel configuration from list of DaqChannel objects.
Use cases:
- Update channel types, sensitivities etc.
Args:
chchfg: New channel configuration
"""
if len(chcfg) != self.nchannels:
raise RuntimeError("Invalid number of channels")
chname = []
sens = []
qtys = []
@ -818,8 +1046,9 @@ class Measurement:
force: If True, overwrites existing files with specified `mfn`
name.
"""
if os.path.splitext(mfn)[1] != ".h5":
mfn += ".h5"
if os.path.splitext(mfn)[1] != DOTMEXT:
mfn += DOTMEXT
if os.path.exists(mfn) and not force:
raise ValueError(f"File {mfn} already exist.")
if timestamp is None:
@ -916,3 +1145,4 @@ class Measurement:
ad[0] = data
return Measurement(newfn)