added flowchart node for removing periodic noise from waveform

metaarray updates:
  - better handling of HDF5 files
  - fixed some isinstance problems that appear during reloads

examples/Flowchart.py

@@ -17,6 +17,7 @@ from pyqtgraph.flowchart import Flowchart
 from pyqtgraph.Qt import QtGui, QtCore
 import pyqtgraph as pg
 import numpy as np
+import pyqtgraph.metaarray as metaarray
 
 app = QtGui.QApplication([])
 
@@ -46,6 +47,7 @@ win.show()
 data = np.random.normal(size=1000)
 data[200:300] += 1
 data += np.sin(np.linspace(0, 100, 1000))
+data = metaarray.MetaArray(data, info=[{'name': 'Time', 'values': np.linspace(0, 1.0, len(data))}, {}])
 
 fc.setInput(dataIn=data)
 

flowchart/library/Data.py

@@ -10,12 +10,6 @@ from pyqtgraph.graphicsItems.LinearRegionItem import LinearRegionItem
 
 from . import functions
 
-try:
-    import metaarray
-    HAVE_METAARRAY = True
-except:
-    HAVE_METAARRAY = False
-
 class ColumnSelectNode(Node):
     """Select named columns from a record array or MetaArray."""
     nodeName = "ColumnSelect"
@@ -31,7 +25,7 @@ class ColumnSelectNode(Node):
             self.updateList(In)
                 
         out = {}
-        if HAVE_METAARRAY and isinstance(In, metaarray.MetaArray):
+        if hasattr(In, 'implements') and In.implements('MetaArray'):
             for c in self.columns:
                 out[c] = In[self.axis:c]
         elif isinstance(In, np.ndarray) and In.dtype.fields is not None:
@@ -47,7 +41,7 @@ class ColumnSelectNode(Node):
         return self.columnList
 
     def updateList(self, data):
-        if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray):
+        if hasattr(data, 'implements') and data.implements('MetaArray'):
             cols = data.listColumns()
             for ax in cols:  ## find first axis with columns
                 if len(cols[ax]) > 0:
@@ -161,7 +155,7 @@ class RegionSelectNode(CtrlNode):
         if self.selected.isConnected():
             if data is None:
                 sliced = None
-            elif isinstance(data, MetaArray):
+            elif (hasattr(data, 'implements') and data.implements('MetaArray')):
                 sliced = data[0:s['start']:s['stop']]
             else:
                 mask = (data['time'] >= s['start']) * (data['time'] < s['stop'])

flowchart/library/Filters.py

@@ -145,7 +145,7 @@ class Derivative(CtrlNode):
     nodeName = 'DerivativeFilter'
     
     def processData(self, data):
-        if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray):
+        if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             info = data.infoCopy()
             if 'values' in info[0]:
                 info[0]['values'] = info[0]['values'][:-1]
@@ -198,3 +198,54 @@ class HistogramDetrend(CtrlNode):
 
 
     
+class RemovePeriodic(CtrlNode):
+    nodeName = 'RemovePeriodic'
+    uiTemplate = [
+        #('windowSize', 'intSpin', {'value': 500, 'min': 10, 'max': 1000000, 'suffix': 'pts'}),
+        #('numBins', 'intSpin', {'value': 50, 'min': 3, 'max': 1000000})
+        ('f0', 'spin', {'value': 60, 'suffix': 'Hz', 'siPrefix': True, 'min': 0, 'max': None}),
+        ('harmonics', 'intSpin', {'value': 30, 'min': 0}),
+    ]
+
+    def processData(self, data):
+        times = data.xvals('Time')
+        dt = times[1]-times[0]
+        
+        data1 = data.asarray()
+        ft = np.fft.fft(data1)
+        
+        ## determine frequencies in fft data
+        df = 1.0 / (len(data1) * dt)
+        freqs = np.linspace(0.0, (len(ft)-1) * df, len(ft))
+        
+        ## flatten spikes at f0 and harmonics
+        f0 = self.ctrls['f0'].value()
+        for i in xrange(1, self.ctrls['harmonics'].value()+2):
+            f = f0 * i # target frequency
+            
+            ## determine index range to check for this frequency
+            ind1 = int(np.floor(f / df))
+            ind2 = int(np.ceil(f / df))
+            if ind1 > len(ft)/2.:
+                break
+            print "--->", f
+            print ind1, ind2, abs(ft[ind1-2:ind2+2])
+            print ft[ind1-2:ind2+2]
+            mag = (abs(ft[ind1-1]) + abs(ft[ind2+1])) * 0.5
+            print "new mag:", mag
+            for j in range(ind1, ind2+1):
+                phase = np.angle(ft[j])
+                re = mag * np.cos(phase)
+                im = mag * np.sin(phase)
+                ft[j] = re + im*1j
+                ft[len(ft)-j] = re - im*1j
+            print abs(ft[ind1-2:ind2+2])
+            print ft[ind1-2:ind2+2]
+                
+        data2 = np.fft.ifft(ft).real
+        
+        ma = metaarray.MetaArray(data2, info=data.infoCopy())
+        return ma
+        
+        
+        

flowchart/library/common.py

@@ -134,7 +134,7 @@ class CtrlNode(Node):
 
 def metaArrayWrapper(fn):
     def newFn(self, data, *args, **kargs):
-        if HAVE_METAARRAY and isinstance(data, metaarray.MetaArray):
+        if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             d1 = fn(self, data.view(np.ndarray), *args, **kargs)
             info = data.infoCopy()
             if d1.shape != data.shape:

flowchart/library/functions.py

@@ -9,7 +9,7 @@ def downsample(data, n, axis=0, xvals='subsample'):
     or downsampled to match.
     """
     ma = None
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         ma = data
         data = data.view(np.ndarray)
         
@@ -60,7 +60,7 @@ def applyFilter(data, b, a, padding=100, bidir=True):
     if padding > 0:
         d1 = d1[padding:-padding]
         
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         return MetaArray(d1, info=data.infoCopy())
     else:
         return d1
@@ -79,7 +79,7 @@ def besselFilter(data, cutoff, order=1, dt=None, btype='low', bidir=True):
     return applyFilter(data, b, a, bidir=bidir)
     #base = data.mean()
     #d1 = scipy.signal.lfilter(b, a, data.view(ndarray)-base) + base
-    #if isinstance(data, MetaArray):
+    #if (hasattr(data, 'implements') and data.implements('MetaArray')):
         #return MetaArray(d1, info=data.infoCopy())
     #return d1
 
@@ -142,7 +142,7 @@ def modeFilter(data, window=500, step=None, bins=None):
     chunks.append(np.linspace(vals[-1], vals[-1], remain))
     d2 = np.hstack(chunks)
     
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         return MetaArray(d2, info=data.infoCopy())
     return d2
 
@@ -169,7 +169,7 @@ def denoise(data, radius=2, threshold=4):
     d6[:radius] = d1[:radius]
     d6[-radius:] = d1[-radius:]
     
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         return MetaArray(d6, info=data.infoCopy())
     return d6
 
@@ -191,7 +191,7 @@ def adaptiveDetrend(data, x=None, threshold=3.0):
     base = lr[1] + lr[0]*x
     d4 = d - base
     
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         return MetaArray(d4, info=data.infoCopy())
     return d4
     
@@ -214,7 +214,7 @@ def histogramDetrend(data, window=500, bins=50, threshold=3.0):
     base = np.linspace(v[0], v[1], len(data))
     d3 = data.view(np.ndarray) - base
     
-    if isinstance(data, MetaArray):
+    if (hasattr(data, 'implements') and data.implements('MetaArray')):
         return MetaArray(d3, info=data.infoCopy())
     return d3
     

graphicsItems/MultiPlotItem.py

@@ -26,7 +26,7 @@ class MultiPlotItem(GraphicsLayout.GraphicsLayout):
         #self.layout.clear()
         self.plots = []
             
-        if HAVE_METAARRAY and isinstance(data, MetaArray):
+        if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             if data.ndim != 2:
                 raise Exception("MultiPlot currently only accepts 2D MetaArray.")
             ic = data.infoCopy()

graphicsItems/PlotItem/PlotItem.py

@@ -887,7 +887,7 @@ class PlotItem(GraphicsWidget):
             
         if params is None:
             params = {}
-        #if HAVE_METAARRAY and isinstance(data, MetaArray):
+        #if HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             #curve = self._plotMetaArray(data, x=x, **kargs)
         #elif isinstance(data, np.ndarray):
             #curve = self._plotArray(data, x=x, **kargs)

imageview/ImageView.py

@@ -29,11 +29,11 @@ import pyqtgraph.debug as debug
 
 from pyqtgraph.SignalProxy import SignalProxy
 
-try:
+#try:
-    import pyqtgraph.metaarray as metaarray
+    #import pyqtgraph.metaarray as metaarray
-    HAVE_METAARRAY = True
+    #HAVE_METAARRAY = True
-except:
+#except:
-    HAVE_METAARRAY = False
+    #HAVE_METAARRAY = False
     
 
 class PlotROI(ROI):
@@ -195,7 +195,7 @@ class ImageView(QtGui.QWidget):
         """
         prof = debug.Profiler('ImageView.setImage', disabled=True)
         
-        if HAVE_METAARRAY and isinstance(img, metaarray.MetaArray):
+        if hasattr(img, 'implements') and img.implements('MetaArray'):
             img = img.asarray()
         
         if not isinstance(img, np.ndarray):

metaarray/MetaArray.py

@@ -4,7 +4,7 @@ MetaArray.py -  Class encapsulating ndarray with meta data
 Copyright 2010  Luke Campagnola
 Distributed under MIT/X11 license. See license.txt for more infomation.
 
-MetaArray is an extension of ndarray which allows storage of per-axis meta data
+MetaArray is an array class based on numpy.ndarray that allows storage of per-axis meta data
 such as axis values, names, units, column names, etc. It also enables several
 new methods for slicing and indexing the array based on this meta data. 
 More info at http://www.scipy.org/Cookbook/MetaArray
@@ -126,7 +126,7 @@ class MetaArray(object):
                 raise Exception("File read failed: %s" % file)
         else:
             self._info = info
-            if isinstance(data, MetaArray):
+            if (hasattr(data, 'implements') and data.implements('MetaArray')):
                 self._info = data._info
                 self._data = data.asarray()
             elif isinstance(data, tuple):  ## create empty array with specified shape
@@ -173,6 +173,13 @@ class MetaArray(object):
                     if len(info[i]['cols']) != self.shape[i]:
                         raise Exception('Length of column list for axis %d does not match data. (given %d, but should be %d)' % (i, len(info[i]['cols']), self.shape[i]))
    
+    def implements(self, name=None):
+        ## Rather than isinstance(obj, MetaArray) use object.implements('MetaArray')
+        if name is None:
+            return ['MetaArray']
+        else:
+            return name == 'MetaArray'
+    
     #def __array_finalize__(self,obj):
         ### array_finalize is called every time a MetaArray is created 
         ### (whereas __new__ is not necessarily called every time)
@@ -670,7 +677,18 @@ class MetaArray(object):
 
     #### File I/O Routines
     def readFile(self, filename, **kwargs):
-        """Load the data and meta info stored in *filename*"""
+        """Load the data and meta info stored in *filename*
+        Different arguments are allowed depending on the type of file.
+        For HDF5 files:
+        
+            *writable* (bool) if True, then any modifications to data in the array will be stored to disk.
+            *readAllData* (bool) if True, then all data in the array is immediately read from disk
+                          and the file is closed (this is the default for files < 500MB). Otherwise, the file will
+                          be left open and data will be read only as requested (this is 
+                          the default for files >= 500MB).
+        
+        
+        """
         ## decide which read function to use
         fd = open(filename, 'rb')
         magic = fd.read(8)
@@ -833,27 +851,39 @@ class MetaArray(object):
         #raise Exception()  ## stress-testing
         #return subarr
 
-    def _readHDF5(self, fileName, close=False, writable=False):
+    def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
+        if 'close' in kargs and readAllData is None: ## for backward compatibility
+            readAllData = kargs['close']
+       
         if not HAVE_HDF5:
             raise Exception("The file '%s' is HDF5-formatted, but the HDF5 library (h5py) was not found." % fileName)
-        f = h5py.File(fileName, 'r')
+        
+        if readAllData is True and writable is True:
+            raise Exception("Incompatible arguments: readAllData=True and writable=True")
+        
+        ## by default, readAllData=True for files < 500MB
+        if readAllData is None:
+            size = os.stat(fileName).st_size
+            readAllData = (size < 500e6)
+        
+        if writable is True:
+            mode = 'r+'
+        else:
+            mode = 'r'
+        f = h5py.File(fileName, mode)
+        
         ver = f.attrs['MetaArray']
         if ver > MetaArray.version:
             print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
         meta = MetaArray.readHDF5Meta(f['info'])
         self._info = meta
         
-        if close:
+        if writable or not readAllData:  ## read all data, convert to ndarray, close file
-            self._data = f['data'][:]
-            f.close()
-        else:
             self._data = f['data']
             self._openFile = f
-        #meta = H5MetaList(f['info'])
+        else:
-        #subarr = arr.view(subtype)
+            self._data = f['data'][:]
-        #subarr._info = meta
+            f.close()
-        #self._data = arr
-        #return subarr
 
     @staticmethod
     def mapHDF5Array(data, writable=False):

widgets/DataTreeWidget.py

@@ -50,7 +50,7 @@ class DataTreeWidget(QtGui.QTreeWidget):
         
         if isinstance(data, types.TracebackType):  ## convert traceback to a list of strings
             data = list(map(str.strip, traceback.format_list(traceback.extract_tb(data))))
-        elif HAVE_METAARRAY and isinstance(data, metaarray.MetaArray):
+        elif HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             data = {
                 'data': data.view(np.ndarray),
                 'meta': data.infoCopy()

widgets/TableWidget.py

@@ -94,7 +94,7 @@ class TableWidget(QtGui.QTableWidget):
             return lambda d: d.__iter__(), None
         elif isinstance(data, dict):
             return lambda d: iter(d.values()), list(map(str, data.keys()))
-        elif HAVE_METAARRAY and isinstance(data, metaarray.MetaArray):
+        elif HAVE_METAARRAY and (hasattr(data, 'implements') and data.implements('MetaArray')):
             if data.axisHasColumns(0):
                 header = [str(data.columnName(0, i)) for i in range(data.shape[0])]
             elif data.axisHasValues(0):