From ae61d3582e8b80eb0ec4d446f3e03767a29b21a6 Mon Sep 17 00:00:00 2001 From: "Paul B. Manis" Date: Tue, 24 Jul 2018 20:01:27 -0400 Subject: [PATCH 1/8] Py2/3 MetaArray adjustments, first pass --- pyqtgraph/metaarray/MetaArray.py | 42 +++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/pyqtgraph/metaarray/MetaArray.py b/pyqtgraph/metaarray/MetaArray.py index 690ff49d..cecea39f 100644 --- a/pyqtgraph/metaarray/MetaArray.py +++ b/pyqtgraph/metaarray/MetaArray.py @@ -26,7 +26,9 @@ except: USE_HDF5 = False HAVE_HDF5 = False - +if HAVE_HDF5: + import h5py.highlevel + def axis(name=None, cols=None, values=None, units=None): """Convenience function for generating axis descriptions when defining MetaArrays""" ax = {} @@ -102,7 +104,7 @@ class MetaArray(object): since the actual values are described (name and units) in the column info for the first axis. """ - version = '2' + version = u'2' # Default hdf5 compression to use when writing # 'gzip' is widely available and somewhat slow @@ -740,7 +742,7 @@ class MetaArray(object): ## decide which read function to use with open(filename, 'rb') as fd: magic = fd.read(8) - if magic == '\x89HDF\r\n\x1a\n': + if magic == b'\x89HDF\r\n\x1a\n': fd.close() self._readHDF5(filename, **kwargs) self._isHDF = True @@ -765,7 +767,7 @@ class MetaArray(object): """Read meta array from the top of a file. Read lines until a blank line is reached. This function should ideally work for ALL versions of MetaArray. """ - meta = '' + meta = u'' ## Read meta information until the first blank line while True: line = fd.readline().strip() @@ -776,6 +778,20 @@ class MetaArray(object): #print ret return ret + def fix_info(self, info): + """ + Recursive version + """ + if isinstance(info, list): + for i in range(len(info)): + info[i] = self.fix_info(info[i]) + elif isinstance(info, dict): + for k in info.keys(): + info[k] = self.fix_info(info[k]) + elif isinstance(info, bytes): # change all bytestrings to string and remove internal quotes + info = info.decode('utf-8').replace("\'", '') + return info + def _readData1(self, fd, meta, mmap=False, **kwds): ## Read array data from the file descriptor for MetaArray v1 files ## read in axis values for any axis that specifies a length @@ -786,7 +802,7 @@ class MetaArray(object): frameSize *= ax['values_len'] del ax['values_len'] del ax['values_type'] - self._info = meta['info'] + self._info = self.fix_info(meta['info']) if not kwds.get("readAllData", True): return ## the remaining data is the actual array @@ -814,7 +830,7 @@ class MetaArray(object): frameSize *= ax['values_len'] del ax['values_len'] del ax['values_type'] - self._info = meta['info'] + self._info = self.fix_info(meta['info']) if not kwds.get("readAllData", True): return @@ -901,7 +917,7 @@ class MetaArray(object): del ax['values_type'] #subarr = subarr.view(subtype) #subarr._info = meta['info'] - self._info = meta['info'] + self._info = self.fix_info(meta['info']) self._data = subarr #raise Exception() ## stress-testing #return subarr @@ -934,10 +950,14 @@ class MetaArray(object): f = h5py.File(fileName, mode) ver = f.attrs['MetaArray'] + try: + ver = ver.decode('utf-8') + except: + pass if ver > MetaArray.version: print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version))) meta = MetaArray.readHDF5Meta(f['info']) - self._info = meta + self._info = self.fix_info(meta) if writable or not readAllData: ## read all data, convert to ndarray, close file self._data = f['data'] @@ -962,7 +982,7 @@ class MetaArray(object): MetaArray._h5py_metaarray = proc._import('pyqtgraph.metaarray') ma = MetaArray._h5py_metaarray.MetaArray(file=fileName) self._data = ma.asarray()._getValue() - self._info = ma._info._getValue() + self._info = self.fix_info(ma._info._getValue()) #print MetaArray._hdf5Process #import inspect #print MetaArray, id(MetaArray), inspect.getmodule(MetaArray) @@ -1010,6 +1030,10 @@ class MetaArray(object): data[k] = val typ = root.attrs['_metaType_'] + try: + typ = typ.decode('utf-8') + except: + pass del data['_metaType_'] if typ == 'dict': From c484c8641710b82acfdb255f124f9534407108c0 Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Wed, 1 Aug 2018 08:57:47 -0700 Subject: [PATCH 2/8] don't modify info from v1 files, move info correction to hdf reading --- pyqtgraph/metaarray/MetaArray.py | 67 ++++++++++---------------------- 1 file changed, 20 insertions(+), 47 deletions(-) diff --git a/pyqtgraph/metaarray/MetaArray.py b/pyqtgraph/metaarray/MetaArray.py index cecea39f..6ce9b05b 100644 --- a/pyqtgraph/metaarray/MetaArray.py +++ b/pyqtgraph/metaarray/MetaArray.py @@ -14,21 +14,19 @@ import types, copy, threading, os, re import pickle import numpy as np from ..python2_3 import basestring -#import traceback + ## By default, the library will use HDF5 when writing files. ## This can be overridden by setting USE_HDF5 = False USE_HDF5 = True try: - import h5py + import h5py.highlevel HAVE_HDF5 = True except: USE_HDF5 = False HAVE_HDF5 = False -if HAVE_HDF5: - import h5py.highlevel - + def axis(name=None, cols=None, values=None, units=None): """Convenience function for generating axis descriptions when defining MetaArrays""" ax = {} @@ -777,20 +775,6 @@ class MetaArray(object): ret = eval(meta) #print ret return ret - - def fix_info(self, info): - """ - Recursive version - """ - if isinstance(info, list): - for i in range(len(info)): - info[i] = self.fix_info(info[i]) - elif isinstance(info, dict): - for k in info.keys(): - info[k] = self.fix_info(info[k]) - elif isinstance(info, bytes): # change all bytestrings to string and remove internal quotes - info = info.decode('utf-8').replace("\'", '') - return info def _readData1(self, fd, meta, mmap=False, **kwds): ## Read array data from the file descriptor for MetaArray v1 files @@ -802,7 +786,7 @@ class MetaArray(object): frameSize *= ax['values_len'] del ax['values_len'] del ax['values_type'] - self._info = self.fix_info(meta['info']) + self._info = meta['info'] if not kwds.get("readAllData", True): return ## the remaining data is the actual array @@ -830,7 +814,7 @@ class MetaArray(object): frameSize *= ax['values_len'] del ax['values_len'] del ax['values_type'] - self._info = self.fix_info(meta['info']) + self._info = meta['info'] if not kwds.get("readAllData", True): return @@ -901,10 +885,8 @@ class MetaArray(object): newSubset = list(subset[:]) newSubset[dynAxis] = slice(dStart, dStop) if dStop > dStart: - #print n, data.shape, " => ", newSubset, data[tuple(newSubset)].shape frames.append(data[tuple(newSubset)].copy()) else: - #data = data[subset].copy() ## what's this for?? frames.append(data) n += inf['numFrames'] @@ -915,12 +897,8 @@ class MetaArray(object): ax['values'] = np.array(xVals, dtype=ax['values_type']) del ax['values_len'] del ax['values_type'] - #subarr = subarr.view(subtype) - #subarr._info = meta['info'] - self._info = self.fix_info(meta['info']) + self._info = meta['info'] self._data = subarr - #raise Exception() ## stress-testing - #return subarr def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs): if 'close' in kargs and readAllData is None: ## for backward compatibility @@ -957,7 +935,7 @@ class MetaArray(object): if ver > MetaArray.version: print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version))) meta = MetaArray.readHDF5Meta(f['info']) - self._info = self.fix_info(meta) + self._info = meta if writable or not readAllData: ## read all data, convert to ndarray, close file self._data = f['data'] @@ -982,12 +960,7 @@ class MetaArray(object): MetaArray._h5py_metaarray = proc._import('pyqtgraph.metaarray') ma = MetaArray._h5py_metaarray.MetaArray(file=fileName) self._data = ma.asarray()._getValue() - self._info = self.fix_info(ma._info._getValue()) - #print MetaArray._hdf5Process - #import inspect - #print MetaArray, id(MetaArray), inspect.getmodule(MetaArray) - - + self._info = ma._info._getValue() @staticmethod def mapHDF5Array(data, writable=False): @@ -999,9 +972,6 @@ class MetaArray(object): if off is None: raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)") return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode) - - - @staticmethod def readHDF5Meta(root, mmap=False): @@ -1010,6 +980,8 @@ class MetaArray(object): ## Pull list of values from attributes and child objects for k in root.attrs: val = root.attrs[k] + if isinstance(val, bytes): + val = val.decode() if isinstance(val, basestring): ## strings need to be re-evaluated to their original types try: val = eval(val) @@ -1047,21 +1019,24 @@ class MetaArray(object): return d2 else: raise Exception("Don't understand metaType '%s'" % typ) - - def write(self, fileName, **opts): + def write(self, fileName, version=2, **opts): """Write this object to a file. The object can be restored by calling MetaArray(file=fileName) opts: appendAxis: the name (or index) of the appendable axis. Allows the array to grow. appendKeys: a list of keys (other than "values") for metadata to append to on the appendable axis. compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc. chunks: bool or tuple specifying chunk shape - """ - - if USE_HDF5 and HAVE_HDF5: + """ + if version == 1: + return self.writeMa(fileName, **opts) + elif USE_HDF5 and HAVE_HDF5: return self.writeHDF5(fileName, **opts) else: - return self.writeMa(fileName, **opts) + if not HAVE_HDF5: + raise Exception("h5py is required for writing .ma version 2 files") + else: + raise Exception("HDF5 is required for writing .ma version 2 files, but it has been disabled.") def writeMeta(self, fileName): """Used to re-write meta info to the given file. @@ -1074,7 +1049,6 @@ class MetaArray(object): self.writeHDF5Meta(f, 'info', self._info) f.close() - def writeHDF5(self, fileName, **opts): ## default options for writing datasets comp = self.defaultCompression @@ -1110,8 +1084,7 @@ class MetaArray(object): ## update options if they were passed in for k in dsOpts: if k in opts: - dsOpts[k] = opts[k] - + dsOpts[k] = opts[k] ## If mappable is in options, it disables chunking/compression if opts.get('mappable', False): From e58b7d4708290ae9af242dc1fd3d879677ebe2d5 Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Wed, 1 Aug 2018 09:02:48 -0700 Subject: [PATCH 3/8] minor correction --- pyqtgraph/metaarray/MetaArray.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pyqtgraph/metaarray/MetaArray.py b/pyqtgraph/metaarray/MetaArray.py index 6ce9b05b..63aee2ec 100644 --- a/pyqtgraph/metaarray/MetaArray.py +++ b/pyqtgraph/metaarray/MetaArray.py @@ -1020,7 +1020,7 @@ class MetaArray(object): else: raise Exception("Don't understand metaType '%s'" % typ) - def write(self, fileName, version=2, **opts): + def write(self, fileName, **opts): """Write this object to a file. The object can be restored by calling MetaArray(file=fileName) opts: appendAxis: the name (or index) of the appendable axis. Allows the array to grow. @@ -1028,15 +1028,12 @@ class MetaArray(object): compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc. chunks: bool or tuple specifying chunk shape """ - if version == 1: + if USE_HDF5 is False: return self.writeMa(fileName, **opts) - elif USE_HDF5 and HAVE_HDF5: + elif HAVE_HDF5 is True: return self.writeHDF5(fileName, **opts) else: - if not HAVE_HDF5: - raise Exception("h5py is required for writing .ma version 2 files") - else: - raise Exception("HDF5 is required for writing .ma version 2 files, but it has been disabled.") + raise Exception("h5py is required for writing .ma hdf5 files, but it could not be imported.") def writeMeta(self, fileName): """Used to re-write meta info to the given file. From 542f4b446b98e26a08d1a815e0534578a06ed0d1 Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Fri, 3 Aug 2018 18:18:28 -0700 Subject: [PATCH 4/8] Add eq() support for comparing dict, list, tuple --- pyqtgraph/functions.py | 20 ++++++++++++++++++++ pyqtgraph/tests/test_functions.py | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/pyqtgraph/functions.py b/pyqtgraph/functions.py index 5cbb177e..ef2d7449 100644 --- a/pyqtgraph/functions.py +++ b/pyqtgraph/functions.py @@ -424,6 +424,8 @@ def eq(a, b): 3. When comparing arrays, returns False if the array shapes are not the same. 4. When comparing arrays of the same shape, returns True only if all elements are equal (whereas the == operator would return a boolean array). + 5. Collections (dict, list, etc.) must have the same type to be considered equal. One + consequence is that comparing a dict to an OrderedDict will always return False. """ if a is b: return True @@ -440,6 +442,24 @@ def eq(a, b): if aIsArr and bIsArr and (a.shape != b.shape or a.dtype != b.dtype): return False + # Recursively handle common containers + if isinstance(a, dict) and isinstance(b, dict): + if type(a) != type(b) or len(a) != len(b): + return False + if a.keys() != b.keys(): + return False + for k,v in a.items(): + if not eq(v, b[k]): + return False + return True + if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): + if type(a) != type(b) or len(a) != len(b): + return False + for v1,v2 in zip(a, b): + if not eq(v1, v2): + return False + return True + # Test for equivalence. # If the test raises a recognized exception, then return Falase try: diff --git a/pyqtgraph/tests/test_functions.py b/pyqtgraph/tests/test_functions.py index e013fe42..fcd16254 100644 --- a/pyqtgraph/tests/test_functions.py +++ b/pyqtgraph/tests/test_functions.py @@ -1,11 +1,15 @@ import pyqtgraph as pg import numpy as np import sys +from copy import deepcopy +from collections import OrderedDict from numpy.testing import assert_array_almost_equal, assert_almost_equal import pytest + np.random.seed(12345) + def testSolve3D(): p1 = np.array([[0,0,0,1], [1,0,0,1], @@ -356,6 +360,29 @@ def test_eq(): assert eq(a4, a4.copy()) assert not eq(a4, a4.T) + # test containers + + assert not eq({'a': 1}, {'a': 1, 'b': 2}) + assert not eq({'a': 1}, {'a': 2}) + d1 = {'x': 1, 'y': np.nan, 3: ['a', np.nan, a3, 7, 2.3], 4: a4} + d2 = deepcopy(d1) + assert eq(d1, d2) + assert eq(OrderedDict(d1), OrderedDict(d2)) + assert not eq(OrderedDict(d1), d2) + items = list(d1.items()) + assert not eq(OrderedDict(items), OrderedDict(reversed(items))) + + assert not eq([1,2,3], [1,2,3,4]) + l1 = [d1, np.inf, -np.inf, np.nan] + l2 = deepcopy(l1) + t1 = tuple(l1) + t2 = tuple(l2) + assert eq(l1, l2) + assert eq(t1, t2) + + assert eq(set(range(10)), set(range(10))) + assert not eq(set(range(10)), set(range(9))) + if __name__ == '__main__': test_interpolateArray() \ No newline at end of file From 7cb27594a5ec54c23b3eca2b6127975af757eac9 Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Mon, 6 Aug 2018 09:05:50 -0700 Subject: [PATCH 5/8] fix dict keys comparison --- pyqtgraph/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyqtgraph/functions.py b/pyqtgraph/functions.py index ef2d7449..8a29d107 100644 --- a/pyqtgraph/functions.py +++ b/pyqtgraph/functions.py @@ -446,7 +446,7 @@ def eq(a, b): if isinstance(a, dict) and isinstance(b, dict): if type(a) != type(b) or len(a) != len(b): return False - if a.keys() != b.keys(): + if set(a.keys()) != set(b.keys()): return False for k,v in a.items(): if not eq(v, b[k]): From a8529e48f35a6cc55ec68b53bcd6a7b37deb0b52 Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Mon, 6 Aug 2018 09:17:23 -0700 Subject: [PATCH 6/8] faster keys comparison --- pyqtgraph/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyqtgraph/functions.py b/pyqtgraph/functions.py index 8a29d107..ef959466 100644 --- a/pyqtgraph/functions.py +++ b/pyqtgraph/functions.py @@ -446,7 +446,7 @@ def eq(a, b): if isinstance(a, dict) and isinstance(b, dict): if type(a) != type(b) or len(a) != len(b): return False - if set(a.keys()) != set(b.keys()): + if sorted(a.keys()) != sorted(b.keys()): return False for k,v in a.items(): if not eq(v, b[k]): From 477feb777bfa31c6427351208df7adeced489b1c Mon Sep 17 00:00:00 2001 From: Ogi Moore Date: Wed, 20 Nov 2019 21:22:31 -0800 Subject: [PATCH 7/8] import h5py.highlevel is deprecated, use import h5py instead --- pyqtgraph/metaarray/MetaArray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyqtgraph/metaarray/MetaArray.py b/pyqtgraph/metaarray/MetaArray.py index 63aee2ec..1410e40c 100644 --- a/pyqtgraph/metaarray/MetaArray.py +++ b/pyqtgraph/metaarray/MetaArray.py @@ -20,7 +20,7 @@ from ..python2_3 import basestring ## This can be overridden by setting USE_HDF5 = False USE_HDF5 = True try: - import h5py.highlevel + import h5py HAVE_HDF5 = True except: USE_HDF5 = False From 71c4807559b3a129ee360062257ad50406ca1271 Mon Sep 17 00:00:00 2001 From: Ogi Moore Date: Wed, 20 Nov 2019 21:22:46 -0800 Subject: [PATCH 8/8] fix dict eq() checks --- pyqtgraph/functions.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pyqtgraph/functions.py b/pyqtgraph/functions.py index ef959466..6cf5f98d 100644 --- a/pyqtgraph/functions.py +++ b/pyqtgraph/functions.py @@ -11,6 +11,7 @@ import numpy as np import decimal, re import ctypes import sys, struct +from .pgcollections import OrderedDict from .python2_3 import asUnicode, basestring from .Qt import QtGui, QtCore, QT_LIB from . import getConfigOption, setConfigOptions @@ -446,11 +447,15 @@ def eq(a, b): if isinstance(a, dict) and isinstance(b, dict): if type(a) != type(b) or len(a) != len(b): return False - if sorted(a.keys()) != sorted(b.keys()): + if set(a.keys()) != set(b.keys()): return False - for k,v in a.items(): + for k, v in a.items(): if not eq(v, b[k]): return False + if isinstance(a, OrderedDict) or sys.version_info >= (3, 7): + for a_item, b_item in zip(a.items(), b.items()): + if not eq(a_item, b_item): + return False return True if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)): if type(a) != type(b) or len(a) != len(b):