From a4963f93b78c501725064354304426d85d536770 Mon Sep 17 00:00:00 2001 From: Luke Campagnola <> Date: Mon, 18 Jun 2012 13:45:47 -0400 Subject: [PATCH] added flowchart node for removing periodic noise from waveform metaarray updates: - better handling of HDF5 files - fixed some isinstance problems that appear during reloads --- examples/Flowchart.py | 2 ++ flowchart/library/Data.py | 12 ++----- flowchart/library/Filters.py | 53 ++++++++++++++++++++++++++- flowchart/library/common.py | 2 +- flowchart/library/functions.py | 14 ++++---- graphicsItems/MultiPlotItem.py | 2 +- graphicsItems/PlotItem/PlotItem.py | 2 +- imageview/ImageView.py | 12 +++---- metaarray/MetaArray.py | 58 ++++++++++++++++++++++-------- widgets/DataTreeWidget.py | 2 +- widgets/TableWidget.py | 2 +- 11 files changed, 119 insertions(+), 42 deletions(-) diff --git a/examples/Flowchart.py b/examples/Flowchart.py index 546faffa..2af8ba0b 100644 --- a/examples/Flowchart.py +++ b/examples/Flowchart.py @@ -17,6 +17,7 @@ from pyqtgraph.flowchart import Flowchart from pyqtgraph.Qt import QtGui, QtCore import pyqtgraph as pg import numpy as np +import pyqtgraph.metaarray as metaarray app = QtGui.QApplication([]) @@ -46,6 +47,7 @@ win.show() data = np.random.normal(size=1000) data[200:300] += 1 data += np.sin(np.linspace(0, 100, 1000)) +data = metaarray.MetaArray(data, info=[{'name': 'Time', 'values': np.linspace(0, 1.0, len(data))}, {}]) fc.setInput(dataIn=data) diff --git a/flowchart/library/Data.py b/flowchart/library/Data.py index da3f6b0e..289cbba6 100644 --- a/flowchart/library/Data.py +++ b/flowchart/library/Data.py @@ -10,12 +10,6 @@ from pyqtgraph.graphicsItems.LinearRegionItem import LinearRegionItem from . import functions -try: - import metaarray - HAVE_METAARRAY = True -except: - HAVE_METAARRAY = False - class ColumnSelectNode(Node): """Select named columns from a record array or MetaArray.""" nodeName = "ColumnSelect" @@ -31,7 +25,7 @@ class ColumnSelectNode(Node): self.updateList(In) out = {} - if HAVE_METAARRAY and isinstance(In, metaarray.MetaArray): + if hasattr(In, 'implements') and In.implements('MetaArray'): for c in self.columns: out[c] = In[self.axis:c] elif isinstance(In, np.ndarray) and In.dtype.fields is not None: @@ -47,7 +41,7 @@ class ColumnSelectNode(Node): return self.columnList def updateList(self, data): - if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray): + if hasattr(data, 'implements') and data.implements('MetaArray'): cols = data.listColumns() for ax in cols: ## find first axis with columns if len(cols[ax]) > 0: @@ -161,7 +155,7 @@ class RegionSelectNode(CtrlNode): if self.selected.isConnected(): if data is None: sliced = None - elif isinstance(data, MetaArray): + elif (hasattr(data, 'implements') and data.implements('MetaArray')): sliced = data[0:s['start']:s['stop']] else: mask = (data['time'] >= s['start']) * (data['time'] < s['stop']) diff --git a/flowchart/library/Filters.py b/flowchart/library/Filters.py index 8816ab58..051afb91 100644 --- a/flowchart/library/Filters.py +++ b/flowchart/library/Filters.py @@ -145,7 +145,7 @@ class Derivative(CtrlNode): nodeName = 'DerivativeFilter' def processData(self, data): - if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray): + if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): info = data.infoCopy() if 'values' in info[0]: info[0]['values'] = info[0]['values'][:-1] @@ -198,3 +198,54 @@ class HistogramDetrend(CtrlNode): +class RemovePeriodic(CtrlNode): + nodeName = 'RemovePeriodic' + uiTemplate = [ + #('windowSize', 'intSpin', {'value': 500, 'min': 10, 'max': 1000000, 'suffix': 'pts'}), + #('numBins', 'intSpin', {'value': 50, 'min': 3, 'max': 1000000}) + ('f0', 'spin', {'value': 60, 'suffix': 'Hz', 'siPrefix': True, 'min': 0, 'max': None}), + ('harmonics', 'intSpin', {'value': 30, 'min': 0}), + ] + + def processData(self, data): + times = data.xvals('Time') + dt = times[1]-times[0] + + data1 = data.asarray() + ft = np.fft.fft(data1) + + ## determine frequencies in fft data + df = 1.0 / (len(data1) * dt) + freqs = np.linspace(0.0, (len(ft)-1) * df, len(ft)) + + ## flatten spikes at f0 and harmonics + f0 = self.ctrls['f0'].value() + for i in xrange(1, self.ctrls['harmonics'].value()+2): + f = f0 * i # target frequency + + ## determine index range to check for this frequency + ind1 = int(np.floor(f / df)) + ind2 = int(np.ceil(f / df)) + if ind1 > len(ft)/2.: + break + print "--->", f + print ind1, ind2, abs(ft[ind1-2:ind2+2]) + print ft[ind1-2:ind2+2] + mag = (abs(ft[ind1-1]) + abs(ft[ind2+1])) * 0.5 + print "new mag:", mag + for j in range(ind1, ind2+1): + phase = np.angle(ft[j]) + re = mag * np.cos(phase) + im = mag * np.sin(phase) + ft[j] = re + im*1j + ft[len(ft)-j] = re - im*1j + print abs(ft[ind1-2:ind2+2]) + print ft[ind1-2:ind2+2] + + data2 = np.fft.ifft(ft).real + + ma = metaarray.MetaArray(data2, info=data.infoCopy()) + return ma + + + \ No newline at end of file diff --git a/flowchart/library/common.py b/flowchart/library/common.py index ce7ff68f..65f8c1fd 100644 --- a/flowchart/library/common.py +++ b/flowchart/library/common.py @@ -134,7 +134,7 @@ class CtrlNode(Node): def metaArrayWrapper(fn): def newFn(self, data, *args, **kargs): - if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray): + if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): d1 = fn(self, data.view(np.ndarray), *args, **kargs) info = data.infoCopy() if d1.shape != data.shape: diff --git a/flowchart/library/functions.py b/flowchart/library/functions.py index 37969cea..2b4e3881 100644 --- a/flowchart/library/functions.py +++ b/flowchart/library/functions.py @@ -9,7 +9,7 @@ def downsample(data, n, axis=0, xvals='subsample'): or downsampled to match. """ ma = None - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): ma = data data = data.view(np.ndarray) @@ -60,7 +60,7 @@ def applyFilter(data, b, a, padding=100, bidir=True): if padding > 0: d1 = d1[padding:-padding] - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): return MetaArray(d1, info=data.infoCopy()) else: return d1 @@ -79,7 +79,7 @@ def besselFilter(data, cutoff, order=1, dt=None, btype='low', bidir=True): return applyFilter(data, b, a, bidir=bidir) #base = data.mean() #d1 = scipy.signal.lfilter(b, a, data.view(ndarray)-base) + base - #if isinstance(data, MetaArray): + #if (hasattr(data, 'implements') and data.implements('MetaArray')): #return MetaArray(d1, info=data.infoCopy()) #return d1 @@ -142,7 +142,7 @@ def modeFilter(data, window=500, step=None, bins=None): chunks.append(np.linspace(vals[-1], vals[-1], remain)) d2 = np.hstack(chunks) - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): return MetaArray(d2, info=data.infoCopy()) return d2 @@ -169,7 +169,7 @@ def denoise(data, radius=2, threshold=4): d6[:radius] = d1[:radius] d6[-radius:] = d1[-radius:] - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): return MetaArray(d6, info=data.infoCopy()) return d6 @@ -191,7 +191,7 @@ def adaptiveDetrend(data, x=None, threshold=3.0): base = lr[1] + lr[0]*x d4 = d - base - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): return MetaArray(d4, info=data.infoCopy()) return d4 @@ -214,7 +214,7 @@ def histogramDetrend(data, window=500, bins=50, threshold=3.0): base = np.linspace(v[0], v[1], len(data)) d3 = data.view(np.ndarray) - base - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): return MetaArray(d3, info=data.infoCopy()) return d3 diff --git a/graphicsItems/MultiPlotItem.py b/graphicsItems/MultiPlotItem.py index 690a8d2c..d20467a9 100644 --- a/graphicsItems/MultiPlotItem.py +++ b/graphicsItems/MultiPlotItem.py @@ -26,7 +26,7 @@ class MultiPlotItem(GraphicsLayout.GraphicsLayout): #self.layout.clear() self.plots = [] - if HAVE_METAARRAY and isinstance(data, MetaArray): + if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): if data.ndim != 2: raise Exception("MultiPlot currently only accepts 2D MetaArray.") ic = data.infoCopy() diff --git a/graphicsItems/PlotItem/PlotItem.py b/graphicsItems/PlotItem/PlotItem.py index 00b3333e..19eb4cf4 100644 --- a/graphicsItems/PlotItem/PlotItem.py +++ b/graphicsItems/PlotItem/PlotItem.py @@ -887,7 +887,7 @@ class PlotItem(GraphicsWidget): if params is None: params = {} - #if HAVE_METAARRAY and isinstance(data, MetaArray): + #if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): #curve = self._plotMetaArray(data, x=x, **kargs) #elif isinstance(data, np.ndarray): #curve = self._plotArray(data, x=x, **kargs) diff --git a/imageview/ImageView.py b/imageview/ImageView.py index a847716d..41a43639 100644 --- a/imageview/ImageView.py +++ b/imageview/ImageView.py @@ -29,11 +29,11 @@ import pyqtgraph.debug as debug from pyqtgraph.SignalProxy import SignalProxy -try: - import pyqtgraph.metaarray as metaarray - HAVE_METAARRAY = True -except: - HAVE_METAARRAY = False +#try: + #import pyqtgraph.metaarray as metaarray + #HAVE_METAARRAY = True +#except: + #HAVE_METAARRAY = False class PlotROI(ROI): @@ -195,7 +195,7 @@ class ImageView(QtGui.QWidget): """ prof = debug.Profiler('ImageView.setImage', disabled=True) - if HAVE_METAARRAY and isinstance(img, metaarray.MetaArray): + if hasattr(img, 'implements') and img.implements('MetaArray'): img = img.asarray() if not isinstance(img, np.ndarray): diff --git a/metaarray/MetaArray.py b/metaarray/MetaArray.py index f35984a0..c7935838 100644 --- a/metaarray/MetaArray.py +++ b/metaarray/MetaArray.py @@ -4,7 +4,7 @@ MetaArray.py - Class encapsulating ndarray with meta data Copyright 2010 Luke Campagnola Distributed under MIT/X11 license. See license.txt for more infomation. -MetaArray is an extension of ndarray which allows storage of per-axis meta data +MetaArray is an array class based on numpy.ndarray that allows storage of per-axis meta data such as axis values, names, units, column names, etc. It also enables several new methods for slicing and indexing the array based on this meta data. More info at http://www.scipy.org/Cookbook/MetaArray @@ -126,7 +126,7 @@ class MetaArray(object): raise Exception("File read failed: %s" % file) else: self._info = info - if isinstance(data, MetaArray): + if (hasattr(data, 'implements') and data.implements('MetaArray')): self._info = data._info self._data = data.asarray() elif isinstance(data, tuple): ## create empty array with specified shape @@ -172,6 +172,13 @@ class MetaArray(object): info[i]['cols'] = list(info[i]['cols']) if len(info[i]['cols']) != self.shape[i]: raise Exception('Length of column list for axis %d does not match data. (given %d, but should be %d)' % (i, len(info[i]['cols']), self.shape[i])) + + def implements(self, name=None): + ## Rather than isinstance(obj, MetaArray) use object.implements('MetaArray') + if name is None: + return ['MetaArray'] + else: + return name == 'MetaArray' #def __array_finalize__(self,obj): ### array_finalize is called every time a MetaArray is created @@ -670,7 +677,18 @@ class MetaArray(object): #### File I/O Routines def readFile(self, filename, **kwargs): - """Load the data and meta info stored in *filename*""" + """Load the data and meta info stored in *filename* + Different arguments are allowed depending on the type of file. + For HDF5 files: + + *writable* (bool) if True, then any modifications to data in the array will be stored to disk. + *readAllData* (bool) if True, then all data in the array is immediately read from disk + and the file is closed (this is the default for files < 500MB). Otherwise, the file will + be left open and data will be read only as requested (this is + the default for files >= 500MB). + + + """ ## decide which read function to use fd = open(filename, 'rb') magic = fd.read(8) @@ -833,27 +851,39 @@ class MetaArray(object): #raise Exception() ## stress-testing #return subarr - def _readHDF5(self, fileName, close=False, writable=False): + def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs): + if 'close' in kargs and readAllData is None: ## for backward compatibility + readAllData = kargs['close'] + if not HAVE_HDF5: raise Exception("The file '%s' is HDF5-formatted, but the HDF5 library (h5py) was not found." % fileName) - f = h5py.File(fileName, 'r') + + if readAllData is True and writable is True: + raise Exception("Incompatible arguments: readAllData=True and writable=True") + + ## by default, readAllData=True for files < 500MB + if readAllData is None: + size = os.stat(fileName).st_size + readAllData = (size < 500e6) + + if writable is True: + mode = 'r+' + else: + mode = 'r' + f = h5py.File(fileName, mode) + ver = f.attrs['MetaArray'] if ver > MetaArray.version: print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version))) meta = MetaArray.readHDF5Meta(f['info']) self._info = meta - if close: - self._data = f['data'][:] - f.close() - else: + if writable or not readAllData: ## read all data, convert to ndarray, close file self._data = f['data'] self._openFile = f - #meta = H5MetaList(f['info']) - #subarr = arr.view(subtype) - #subarr._info = meta - #self._data = arr - #return subarr + else: + self._data = f['data'][:] + f.close() @staticmethod def mapHDF5Array(data, writable=False): diff --git a/widgets/DataTreeWidget.py b/widgets/DataTreeWidget.py index 31eea042..9a04d1d2 100644 --- a/widgets/DataTreeWidget.py +++ b/widgets/DataTreeWidget.py @@ -50,7 +50,7 @@ class DataTreeWidget(QtGui.QTreeWidget): if isinstance(data, types.TracebackType): ## convert traceback to a list of strings data = list(map(str.strip, traceback.format_list(traceback.extract_tb(data)))) - elif HAVE_METAARRAY and isinstance(data, metaarray.MetaArray): + elif HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): data = { 'data': data.view(np.ndarray), 'meta': data.infoCopy() diff --git a/widgets/TableWidget.py b/widgets/TableWidget.py index e57e90c3..c58b5cb0 100644 --- a/widgets/TableWidget.py +++ b/widgets/TableWidget.py @@ -94,7 +94,7 @@ class TableWidget(QtGui.QTableWidget): return lambda d: d.__iter__(), None elif isinstance(data, dict): return lambda d: iter(d.values()), list(map(str, data.keys())) - elif HAVE_METAARRAY and isinstance(data, metaarray.MetaArray): + elif HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')): if data.axisHasColumns(0): header = [str(data.columnName(0, i)) for i in range(data.shape[0])] elif data.axisHasValues(0):