Merge pull request #742 from campagnola/metaarray-py3

Metaarray py3 support

pyqtgraph/functions.py

@@ -11,6 +11,7 @@ import numpy as np
 import decimal, re
 import ctypes
 import sys, struct
+from .pgcollections import OrderedDict
 from .python2_3 import asUnicode, basestring
 from .Qt import QtGui, QtCore, QT_LIB
 from . import getConfigOption, setConfigOptions
@@ -424,6 +425,8 @@ def eq(a, b):
     3. When comparing arrays, returns False if the array shapes are not the same.
     4. When comparing arrays of the same shape, returns True only if all elements are equal (whereas
        the == operator would return a boolean array).
+    5. Collections (dict, list, etc.) must have the same type to be considered equal. One 
+       consequence is that comparing a dict to an OrderedDict will always return False. 
     """
     if a is b:
         return True
@@ -440,6 +443,28 @@ def eq(a, b):
     if aIsArr and bIsArr and (a.shape != b.shape or a.dtype != b.dtype):
         return False
 
+    # Recursively handle common containers
+    if isinstance(a, dict) and isinstance(b, dict):
+        if type(a) != type(b) or len(a) != len(b):
+            return False
+        if set(a.keys()) != set(b.keys()):
+            return False
+        for k, v in a.items():
+            if not eq(v, b[k]):
+                return False
+        if isinstance(a, OrderedDict) or sys.version_info >= (3, 7):
+            for a_item, b_item in zip(a.items(), b.items()):
+                if not eq(a_item, b_item):
+                    return False
+        return True
+    if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
+        if type(a) != type(b) or len(a) != len(b):
+            return False
+        for v1,v2 in zip(a, b):
+            if not eq(v1, v2):
+                return False
+        return True
+
     # Test for equivalence. 
     # If the test raises a recognized exception, then return Falase
     try:

pyqtgraph/metaarray/MetaArray.py

@@ -14,7 +14,7 @@ import types, copy, threading, os, re
 import pickle
 import numpy as np
 from ..python2_3 import basestring
-#import traceback
+
 
 ## By default, the library will use HDF5 when writing files.
 ## This can be overridden by setting USE_HDF5 = False
@@ -102,7 +102,7 @@ class MetaArray(object):
         since the actual values are described (name and units) in the column info for the first axis.
     """
   
-    version = '2'
+    version = u'2'
 
     # Default hdf5 compression to use when writing
     #   'gzip' is widely available and somewhat slow
@@ -740,7 +740,7 @@ class MetaArray(object):
         ## decide which read function to use
         with open(filename, 'rb') as fd:
             magic = fd.read(8)
-            if magic == '\x89HDF\r\n\x1a\n':
+            if magic == b'\x89HDF\r\n\x1a\n':
                 fd.close()
                 self._readHDF5(filename, **kwargs)
                 self._isHDF = True
@@ -765,7 +765,7 @@ class MetaArray(object):
         """Read meta array from the top of a file. Read lines until a blank line is reached.
         This function should ideally work for ALL versions of MetaArray.
         """
-        meta = ''
+        meta = u''
         ## Read meta information until the first blank line
         while True:
             line = fd.readline().strip()
@@ -775,7 +775,7 @@ class MetaArray(object):
         ret = eval(meta)
         #print ret
         return ret
-
+    
     def _readData1(self, fd, meta, mmap=False, **kwds):
         ## Read array data from the file descriptor for MetaArray v1 files
         ## read in axis values for any axis that specifies a length
@@ -885,10 +885,8 @@ class MetaArray(object):
                     newSubset = list(subset[:])
                     newSubset[dynAxis] = slice(dStart, dStop)
                     if dStop > dStart:
-                        #print n, data.shape, " => ", newSubset, data[tuple(newSubset)].shape
                         frames.append(data[tuple(newSubset)].copy())
                 else:
-                    #data = data[subset].copy()  ## what's this for??
                     frames.append(data)
                 
                 n += inf['numFrames']
@@ -899,12 +897,8 @@ class MetaArray(object):
                 ax['values'] = np.array(xVals, dtype=ax['values_type'])
             del ax['values_len']
             del ax['values_type']
-        #subarr = subarr.view(subtype)
-        #subarr._info = meta['info']
         self._info = meta['info']
         self._data = subarr
-        #raise Exception()  ## stress-testing
-        #return subarr
 
     def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
         if 'close' in kargs and readAllData is None: ## for backward compatibility
@@ -934,6 +928,10 @@ class MetaArray(object):
         f = h5py.File(fileName, mode)
         
         ver = f.attrs['MetaArray']
+        try:
+            ver = ver.decode('utf-8')
+        except:
+            pass
         if ver > MetaArray.version:
             print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
         meta = MetaArray.readHDF5Meta(f['info'])
@@ -963,11 +961,6 @@ class MetaArray(object):
         ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
         self._data = ma.asarray()._getValue()
         self._info = ma._info._getValue()
-        #print MetaArray._hdf5Process
-        #import inspect
-        #print MetaArray, id(MetaArray), inspect.getmodule(MetaArray)
-        
-        
 
     @staticmethod
     def mapHDF5Array(data, writable=False):
@@ -979,9 +972,6 @@ class MetaArray(object):
         if off is None:
             raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
         return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
-        
-
-
 
     @staticmethod
     def readHDF5Meta(root, mmap=False):
@@ -990,6 +980,8 @@ class MetaArray(object):
         ## Pull list of values from attributes and child objects
         for k in root.attrs:
             val = root.attrs[k]
+            if isinstance(val, bytes):
+                val = val.decode()
             if isinstance(val, basestring):  ## strings need to be re-evaluated to their original types
                 try:
                     val = eval(val)
@@ -1010,6 +1002,10 @@ class MetaArray(object):
             data[k] = val
         
         typ = root.attrs['_metaType_']
+        try:
+            typ = typ.decode('utf-8')
+        except:
+            pass
         del data['_metaType_']
         
         if typ == 'dict':
@@ -1023,7 +1019,6 @@ class MetaArray(object):
             return d2
         else:
             raise Exception("Don't understand metaType '%s'" % typ)
-        
 
     def write(self, fileName, **opts):
         """Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
@@ -1032,12 +1027,13 @@ class MetaArray(object):
             appendKeys: a list of keys (other than "values") for metadata to append to on the appendable axis.
             compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
             chunks: bool or tuple specifying chunk shape
-        """
+        """        
-        
+        if USE_HDF5 is False:
-        if USE_HDF5 and HAVE_HDF5:
+            return self.writeMa(fileName, **opts)
+        elif HAVE_HDF5 is True:
             return self.writeHDF5(fileName, **opts)
         else:
-            return self.writeMa(fileName, **opts)
+            raise Exception("h5py is required for writing .ma hdf5 files, but it could not be imported.")
 
     def writeMeta(self, fileName):
         """Used to re-write meta info to the given file.
@@ -1050,7 +1046,6 @@ class MetaArray(object):
         self.writeHDF5Meta(f, 'info', self._info)
         f.close()
 
-
     def writeHDF5(self, fileName, **opts):
         ## default options for writing datasets
         comp = self.defaultCompression
@@ -1086,8 +1081,7 @@ class MetaArray(object):
         ## update options if they were passed in
         for k in dsOpts:
             if k in opts:
-                dsOpts[k] = opts[k]
+                dsOpts[k] = opts[k]        
-        
         
         ## If mappable is in options, it disables chunking/compression
         if opts.get('mappable', False):

pyqtgraph/tests/test_functions.py

@@ -1,11 +1,15 @@
 import pyqtgraph as pg
 import numpy as np
 import sys
+from copy import deepcopy
+from collections import OrderedDict
 from numpy.testing import assert_array_almost_equal, assert_almost_equal
 import pytest
 
+
 np.random.seed(12345)
 
+
 def testSolve3D():
     p1 = np.array([[0,0,0,1],
                    [1,0,0,1],
@@ -356,6 +360,29 @@ def test_eq():
     assert eq(a4, a4.copy())
     assert not eq(a4, a4.T)
 
+    # test containers
+
+    assert not eq({'a': 1}, {'a': 1, 'b': 2})
+    assert not eq({'a': 1}, {'a': 2})
+    d1 = {'x': 1, 'y': np.nan, 3: ['a', np.nan, a3, 7, 2.3], 4: a4}
+    d2 = deepcopy(d1)
+    assert eq(d1, d2)
+    assert eq(OrderedDict(d1), OrderedDict(d2))
+    assert not eq(OrderedDict(d1), d2)
+    items = list(d1.items())
+    assert not eq(OrderedDict(items), OrderedDict(reversed(items)))
+    
+    assert not eq([1,2,3], [1,2,3,4])
+    l1 = [d1, np.inf, -np.inf, np.nan]
+    l2 = deepcopy(l1)
+    t1 = tuple(l1)
+    t2 = tuple(l2)
+    assert eq(l1, l2)
+    assert eq(t1, t2)
+
+    assert eq(set(range(10)), set(range(10)))
+    assert not eq(set(range(10)), set(range(9)))
+
     
 if __name__ == '__main__':
     test_interpolateArray()