Merge pull request #742 from campagnola/metaarray-py3
Metaarray py3 support
This commit is contained in:
Ogi Moore
GitHub
commit
0f40237012
|
|
@ -11,6 +11,7 @@ import numpy as np
|
|||
import decimal, re
|
||||
import ctypes
|
||||
import sys, struct
|
||||
from .pgcollections import OrderedDict
|
||||
from .python2_3 import asUnicode, basestring
|
||||
from .Qt import QtGui, QtCore, QT_LIB
|
||||
from . import getConfigOption, setConfigOptions
|
||||
|
|
@ -424,6 +425,8 @@ def eq(a, b):
|
|||
3. When comparing arrays, returns False if the array shapes are not the same.
|
||||
4. When comparing arrays of the same shape, returns True only if all elements are equal (whereas
|
||||
the == operator would return a boolean array).
|
||||
5. Collections (dict, list, etc.) must have the same type to be considered equal. One
|
||||
consequence is that comparing a dict to an OrderedDict will always return False.
|
||||
"""
|
||||
if a is b:
|
||||
return True
|
||||
|
|
@ -440,6 +443,28 @@ def eq(a, b):
|
|||
if aIsArr and bIsArr and (a.shape != b.shape or a.dtype != b.dtype):
|
||||
return False
|
||||
|
||||
# Recursively handle common containers
|
||||
if isinstance(a, dict) and isinstance(b, dict):
|
||||
if type(a) != type(b) or len(a) != len(b):
|
||||
return False
|
||||
if set(a.keys()) != set(b.keys()):
|
||||
return False
|
||||
for k, v in a.items():
|
||||
if not eq(v, b[k]):
|
||||
return False
|
||||
if isinstance(a, OrderedDict) or sys.version_info >= (3, 7):
|
||||
for a_item, b_item in zip(a.items(), b.items()):
|
||||
if not eq(a_item, b_item):
|
||||
return False
|
||||
return True
|
||||
if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
|
||||
if type(a) != type(b) or len(a) != len(b):
|
||||
return False
|
||||
for v1,v2 in zip(a, b):
|
||||
if not eq(v1, v2):
|
||||
return False
|
||||
return True
|
||||
|
||||
# Test for equivalence.
|
||||
# If the test raises a recognized exception, then return Falase
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import types, copy, threading, os, re
|
|||
import pickle
|
||||
import numpy as np
|
||||
from ..python2_3 import basestring
|
||||
#import traceback
|
||||
|
||||
|
||||
## By default, the library will use HDF5 when writing files.
|
||||
## This can be overridden by setting USE_HDF5 = False
|
||||
|
|
@ -102,7 +102,7 @@ class MetaArray(object):
|
|||
since the actual values are described (name and units) in the column info for the first axis.
|
||||
"""
|
||||
|
||||
version = '2'
|
||||
version = u'2'
|
||||
|
||||
# Default hdf5 compression to use when writing
|
||||
# 'gzip' is widely available and somewhat slow
|
||||
|
|
@ -740,7 +740,7 @@ class MetaArray(object):
|
|||
## decide which read function to use
|
||||
with open(filename, 'rb') as fd:
|
||||
magic = fd.read(8)
|
||||
if magic == '\x89HDF\r\n\x1a\n':
|
||||
if magic == b'\x89HDF\r\n\x1a\n':
|
||||
fd.close()
|
||||
self._readHDF5(filename, **kwargs)
|
||||
self._isHDF = True
|
||||
|
|
@ -765,7 +765,7 @@ class MetaArray(object):
|
|||
"""Read meta array from the top of a file. Read lines until a blank line is reached.
|
||||
This function should ideally work for ALL versions of MetaArray.
|
||||
"""
|
||||
meta = ''
|
||||
meta = u''
|
||||
## Read meta information until the first blank line
|
||||
while True:
|
||||
line = fd.readline().strip()
|
||||
|
|
@ -775,7 +775,7 @@ class MetaArray(object):
|
|||
ret = eval(meta)
|
||||
#print ret
|
||||
return ret
|
||||
|
||||
|
||||
def _readData1(self, fd, meta, mmap=False, **kwds):
|
||||
## Read array data from the file descriptor for MetaArray v1 files
|
||||
## read in axis values for any axis that specifies a length
|
||||
|
|
@ -885,10 +885,8 @@ class MetaArray(object):
|
|||
newSubset = list(subset[:])
|
||||
newSubset[dynAxis] = slice(dStart, dStop)
|
||||
if dStop > dStart:
|
||||
#print n, data.shape, " => ", newSubset, data[tuple(newSubset)].shape
|
||||
frames.append(data[tuple(newSubset)].copy())
|
||||
else:
|
||||
#data = data[subset].copy() ## what's this for??
|
||||
frames.append(data)
|
||||
|
||||
n += inf['numFrames']
|
||||
|
|
@ -899,12 +897,8 @@ class MetaArray(object):
|
|||
ax['values'] = np.array(xVals, dtype=ax['values_type'])
|
||||
del ax['values_len']
|
||||
del ax['values_type']
|
||||
#subarr = subarr.view(subtype)
|
||||
#subarr._info = meta['info']
|
||||
self._info = meta['info']
|
||||
self._data = subarr
|
||||
#raise Exception() ## stress-testing
|
||||
#return subarr
|
||||
|
||||
def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
|
||||
if 'close' in kargs and readAllData is None: ## for backward compatibility
|
||||
|
|
@ -934,6 +928,10 @@ class MetaArray(object):
|
|||
f = h5py.File(fileName, mode)
|
||||
|
||||
ver = f.attrs['MetaArray']
|
||||
try:
|
||||
ver = ver.decode('utf-8')
|
||||
except:
|
||||
pass
|
||||
if ver > MetaArray.version:
|
||||
print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
|
||||
meta = MetaArray.readHDF5Meta(f['info'])
|
||||
|
|
@ -963,11 +961,6 @@ class MetaArray(object):
|
|||
ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
|
||||
self._data = ma.asarray()._getValue()
|
||||
self._info = ma._info._getValue()
|
||||
#print MetaArray._hdf5Process
|
||||
#import inspect
|
||||
#print MetaArray, id(MetaArray), inspect.getmodule(MetaArray)
|
||||
|
||||
|
||||
|
||||
@staticmethod
|
||||
def mapHDF5Array(data, writable=False):
|
||||
|
|
@ -979,9 +972,6 @@ class MetaArray(object):
|
|||
if off is None:
|
||||
raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
|
||||
return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
|
||||
|
||||
|
||||
|
||||
|
||||
@staticmethod
|
||||
def readHDF5Meta(root, mmap=False):
|
||||
|
|
@ -990,6 +980,8 @@ class MetaArray(object):
|
|||
## Pull list of values from attributes and child objects
|
||||
for k in root.attrs:
|
||||
val = root.attrs[k]
|
||||
if isinstance(val, bytes):
|
||||
val = val.decode()
|
||||
if isinstance(val, basestring): ## strings need to be re-evaluated to their original types
|
||||
try:
|
||||
val = eval(val)
|
||||
|
|
@ -1010,6 +1002,10 @@ class MetaArray(object):
|
|||
data[k] = val
|
||||
|
||||
typ = root.attrs['_metaType_']
|
||||
try:
|
||||
typ = typ.decode('utf-8')
|
||||
except:
|
||||
pass
|
||||
del data['_metaType_']
|
||||
|
||||
if typ == 'dict':
|
||||
|
|
@ -1023,7 +1019,6 @@ class MetaArray(object):
|
|||
return d2
|
||||
else:
|
||||
raise Exception("Don't understand metaType '%s'" % typ)
|
||||
|
||||
|
||||
def write(self, fileName, **opts):
|
||||
"""Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
|
||||
|
|
@ -1032,12 +1027,13 @@ class MetaArray(object):
|
|||
appendKeys: a list of keys (other than "values") for metadata to append to on the appendable axis.
|
||||
compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
|
||||
chunks: bool or tuple specifying chunk shape
|
||||
"""
|
||||
|
||||
if USE_HDF5 and HAVE_HDF5:
|
||||
"""
|
||||
if USE_HDF5 is False:
|
||||
return self.writeMa(fileName, **opts)
|
||||
elif HAVE_HDF5 is True:
|
||||
return self.writeHDF5(fileName, **opts)
|
||||
else:
|
||||
return self.writeMa(fileName, **opts)
|
||||
raise Exception("h5py is required for writing .ma hdf5 files, but it could not be imported.")
|
||||
|
||||
def writeMeta(self, fileName):
|
||||
"""Used to re-write meta info to the given file.
|
||||
|
|
@ -1050,7 +1046,6 @@ class MetaArray(object):
|
|||
self.writeHDF5Meta(f, 'info', self._info)
|
||||
f.close()
|
||||
|
||||
|
||||
def writeHDF5(self, fileName, **opts):
|
||||
## default options for writing datasets
|
||||
comp = self.defaultCompression
|
||||
|
|
@ -1086,8 +1081,7 @@ class MetaArray(object):
|
|||
## update options if they were passed in
|
||||
for k in dsOpts:
|
||||
if k in opts:
|
||||
dsOpts[k] = opts[k]
|
||||
|
||||
dsOpts[k] = opts[k]
|
||||
|
||||
## If mappable is in options, it disables chunking/compression
|
||||
if opts.get('mappable', False):
|
||||
|
|
|
|||
|
|
@ -1,11 +1,15 @@
|
|||
import pyqtgraph as pg
|
||||
import numpy as np
|
||||
import sys
|
||||
from copy import deepcopy
|
||||
from collections import OrderedDict
|
||||
from numpy.testing import assert_array_almost_equal, assert_almost_equal
|
||||
import pytest
|
||||
|
||||
|
||||
np.random.seed(12345)
|
||||
|
||||
|
||||
def testSolve3D():
|
||||
p1 = np.array([[0,0,0,1],
|
||||
[1,0,0,1],
|
||||
|
|
@ -356,6 +360,29 @@ def test_eq():
|
|||
assert eq(a4, a4.copy())
|
||||
assert not eq(a4, a4.T)
|
||||
|
||||
# test containers
|
||||
|
||||
assert not eq({'a': 1}, {'a': 1, 'b': 2})
|
||||
assert not eq({'a': 1}, {'a': 2})
|
||||
d1 = {'x': 1, 'y': np.nan, 3: ['a', np.nan, a3, 7, 2.3], 4: a4}
|
||||
d2 = deepcopy(d1)
|
||||
assert eq(d1, d2)
|
||||
assert eq(OrderedDict(d1), OrderedDict(d2))
|
||||
assert not eq(OrderedDict(d1), d2)
|
||||
items = list(d1.items())
|
||||
assert not eq(OrderedDict(items), OrderedDict(reversed(items)))
|
||||
|
||||
assert not eq([1,2,3], [1,2,3,4])
|
||||
l1 = [d1, np.inf, -np.inf, np.nan]
|
||||
l2 = deepcopy(l1)
|
||||
t1 = tuple(l1)
|
||||
t2 = tuple(l2)
|
||||
assert eq(l1, l2)
|
||||
assert eq(t1, t2)
|
||||
|
||||
assert eq(set(range(10)), set(range(10)))
|
||||
assert not eq(set(range(10)), set(range(9)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_interpolateArray()
|
||||
Loading…
Reference in New Issue
Block a user