Merge pull request #742 from campagnola/metaarray-py3
Metaarray py3 support
This commit is contained in:
commit
0f40237012
@ -11,6 +11,7 @@ import numpy as np
|
|||||||
import decimal, re
|
import decimal, re
|
||||||
import ctypes
|
import ctypes
|
||||||
import sys, struct
|
import sys, struct
|
||||||
|
from .pgcollections import OrderedDict
|
||||||
from .python2_3 import asUnicode, basestring
|
from .python2_3 import asUnicode, basestring
|
||||||
from .Qt import QtGui, QtCore, QT_LIB
|
from .Qt import QtGui, QtCore, QT_LIB
|
||||||
from . import getConfigOption, setConfigOptions
|
from . import getConfigOption, setConfigOptions
|
||||||
@ -424,6 +425,8 @@ def eq(a, b):
|
|||||||
3. When comparing arrays, returns False if the array shapes are not the same.
|
3. When comparing arrays, returns False if the array shapes are not the same.
|
||||||
4. When comparing arrays of the same shape, returns True only if all elements are equal (whereas
|
4. When comparing arrays of the same shape, returns True only if all elements are equal (whereas
|
||||||
the == operator would return a boolean array).
|
the == operator would return a boolean array).
|
||||||
|
5. Collections (dict, list, etc.) must have the same type to be considered equal. One
|
||||||
|
consequence is that comparing a dict to an OrderedDict will always return False.
|
||||||
"""
|
"""
|
||||||
if a is b:
|
if a is b:
|
||||||
return True
|
return True
|
||||||
@ -440,6 +443,28 @@ def eq(a, b):
|
|||||||
if aIsArr and bIsArr and (a.shape != b.shape or a.dtype != b.dtype):
|
if aIsArr and bIsArr and (a.shape != b.shape or a.dtype != b.dtype):
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
# Recursively handle common containers
|
||||||
|
if isinstance(a, dict) and isinstance(b, dict):
|
||||||
|
if type(a) != type(b) or len(a) != len(b):
|
||||||
|
return False
|
||||||
|
if set(a.keys()) != set(b.keys()):
|
||||||
|
return False
|
||||||
|
for k, v in a.items():
|
||||||
|
if not eq(v, b[k]):
|
||||||
|
return False
|
||||||
|
if isinstance(a, OrderedDict) or sys.version_info >= (3, 7):
|
||||||
|
for a_item, b_item in zip(a.items(), b.items()):
|
||||||
|
if not eq(a_item, b_item):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
if isinstance(a, (list, tuple)) and isinstance(b, (list, tuple)):
|
||||||
|
if type(a) != type(b) or len(a) != len(b):
|
||||||
|
return False
|
||||||
|
for v1,v2 in zip(a, b):
|
||||||
|
if not eq(v1, v2):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
# Test for equivalence.
|
# Test for equivalence.
|
||||||
# If the test raises a recognized exception, then return Falase
|
# If the test raises a recognized exception, then return Falase
|
||||||
try:
|
try:
|
||||||
|
@ -14,7 +14,7 @@ import types, copy, threading, os, re
|
|||||||
import pickle
|
import pickle
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from ..python2_3 import basestring
|
from ..python2_3 import basestring
|
||||||
#import traceback
|
|
||||||
|
|
||||||
## By default, the library will use HDF5 when writing files.
|
## By default, the library will use HDF5 when writing files.
|
||||||
## This can be overridden by setting USE_HDF5 = False
|
## This can be overridden by setting USE_HDF5 = False
|
||||||
@ -102,7 +102,7 @@ class MetaArray(object):
|
|||||||
since the actual values are described (name and units) in the column info for the first axis.
|
since the actual values are described (name and units) in the column info for the first axis.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
version = '2'
|
version = u'2'
|
||||||
|
|
||||||
# Default hdf5 compression to use when writing
|
# Default hdf5 compression to use when writing
|
||||||
# 'gzip' is widely available and somewhat slow
|
# 'gzip' is widely available and somewhat slow
|
||||||
@ -740,7 +740,7 @@ class MetaArray(object):
|
|||||||
## decide which read function to use
|
## decide which read function to use
|
||||||
with open(filename, 'rb') as fd:
|
with open(filename, 'rb') as fd:
|
||||||
magic = fd.read(8)
|
magic = fd.read(8)
|
||||||
if magic == '\x89HDF\r\n\x1a\n':
|
if magic == b'\x89HDF\r\n\x1a\n':
|
||||||
fd.close()
|
fd.close()
|
||||||
self._readHDF5(filename, **kwargs)
|
self._readHDF5(filename, **kwargs)
|
||||||
self._isHDF = True
|
self._isHDF = True
|
||||||
@ -765,7 +765,7 @@ class MetaArray(object):
|
|||||||
"""Read meta array from the top of a file. Read lines until a blank line is reached.
|
"""Read meta array from the top of a file. Read lines until a blank line is reached.
|
||||||
This function should ideally work for ALL versions of MetaArray.
|
This function should ideally work for ALL versions of MetaArray.
|
||||||
"""
|
"""
|
||||||
meta = ''
|
meta = u''
|
||||||
## Read meta information until the first blank line
|
## Read meta information until the first blank line
|
||||||
while True:
|
while True:
|
||||||
line = fd.readline().strip()
|
line = fd.readline().strip()
|
||||||
@ -885,10 +885,8 @@ class MetaArray(object):
|
|||||||
newSubset = list(subset[:])
|
newSubset = list(subset[:])
|
||||||
newSubset[dynAxis] = slice(dStart, dStop)
|
newSubset[dynAxis] = slice(dStart, dStop)
|
||||||
if dStop > dStart:
|
if dStop > dStart:
|
||||||
#print n, data.shape, " => ", newSubset, data[tuple(newSubset)].shape
|
|
||||||
frames.append(data[tuple(newSubset)].copy())
|
frames.append(data[tuple(newSubset)].copy())
|
||||||
else:
|
else:
|
||||||
#data = data[subset].copy() ## what's this for??
|
|
||||||
frames.append(data)
|
frames.append(data)
|
||||||
|
|
||||||
n += inf['numFrames']
|
n += inf['numFrames']
|
||||||
@ -899,12 +897,8 @@ class MetaArray(object):
|
|||||||
ax['values'] = np.array(xVals, dtype=ax['values_type'])
|
ax['values'] = np.array(xVals, dtype=ax['values_type'])
|
||||||
del ax['values_len']
|
del ax['values_len']
|
||||||
del ax['values_type']
|
del ax['values_type']
|
||||||
#subarr = subarr.view(subtype)
|
|
||||||
#subarr._info = meta['info']
|
|
||||||
self._info = meta['info']
|
self._info = meta['info']
|
||||||
self._data = subarr
|
self._data = subarr
|
||||||
#raise Exception() ## stress-testing
|
|
||||||
#return subarr
|
|
||||||
|
|
||||||
def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
|
def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
|
||||||
if 'close' in kargs and readAllData is None: ## for backward compatibility
|
if 'close' in kargs and readAllData is None: ## for backward compatibility
|
||||||
@ -934,6 +928,10 @@ class MetaArray(object):
|
|||||||
f = h5py.File(fileName, mode)
|
f = h5py.File(fileName, mode)
|
||||||
|
|
||||||
ver = f.attrs['MetaArray']
|
ver = f.attrs['MetaArray']
|
||||||
|
try:
|
||||||
|
ver = ver.decode('utf-8')
|
||||||
|
except:
|
||||||
|
pass
|
||||||
if ver > MetaArray.version:
|
if ver > MetaArray.version:
|
||||||
print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
|
print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
|
||||||
meta = MetaArray.readHDF5Meta(f['info'])
|
meta = MetaArray.readHDF5Meta(f['info'])
|
||||||
@ -963,11 +961,6 @@ class MetaArray(object):
|
|||||||
ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
|
ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
|
||||||
self._data = ma.asarray()._getValue()
|
self._data = ma.asarray()._getValue()
|
||||||
self._info = ma._info._getValue()
|
self._info = ma._info._getValue()
|
||||||
#print MetaArray._hdf5Process
|
|
||||||
#import inspect
|
|
||||||
#print MetaArray, id(MetaArray), inspect.getmodule(MetaArray)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def mapHDF5Array(data, writable=False):
|
def mapHDF5Array(data, writable=False):
|
||||||
@ -980,9 +973,6 @@ class MetaArray(object):
|
|||||||
raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
|
raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
|
||||||
return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
|
return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def readHDF5Meta(root, mmap=False):
|
def readHDF5Meta(root, mmap=False):
|
||||||
data = {}
|
data = {}
|
||||||
@ -990,6 +980,8 @@ class MetaArray(object):
|
|||||||
## Pull list of values from attributes and child objects
|
## Pull list of values from attributes and child objects
|
||||||
for k in root.attrs:
|
for k in root.attrs:
|
||||||
val = root.attrs[k]
|
val = root.attrs[k]
|
||||||
|
if isinstance(val, bytes):
|
||||||
|
val = val.decode()
|
||||||
if isinstance(val, basestring): ## strings need to be re-evaluated to their original types
|
if isinstance(val, basestring): ## strings need to be re-evaluated to their original types
|
||||||
try:
|
try:
|
||||||
val = eval(val)
|
val = eval(val)
|
||||||
@ -1010,6 +1002,10 @@ class MetaArray(object):
|
|||||||
data[k] = val
|
data[k] = val
|
||||||
|
|
||||||
typ = root.attrs['_metaType_']
|
typ = root.attrs['_metaType_']
|
||||||
|
try:
|
||||||
|
typ = typ.decode('utf-8')
|
||||||
|
except:
|
||||||
|
pass
|
||||||
del data['_metaType_']
|
del data['_metaType_']
|
||||||
|
|
||||||
if typ == 'dict':
|
if typ == 'dict':
|
||||||
@ -1024,7 +1020,6 @@ class MetaArray(object):
|
|||||||
else:
|
else:
|
||||||
raise Exception("Don't understand metaType '%s'" % typ)
|
raise Exception("Don't understand metaType '%s'" % typ)
|
||||||
|
|
||||||
|
|
||||||
def write(self, fileName, **opts):
|
def write(self, fileName, **opts):
|
||||||
"""Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
|
"""Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
|
||||||
opts:
|
opts:
|
||||||
@ -1033,11 +1028,12 @@ class MetaArray(object):
|
|||||||
compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
|
compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
|
||||||
chunks: bool or tuple specifying chunk shape
|
chunks: bool or tuple specifying chunk shape
|
||||||
"""
|
"""
|
||||||
|
if USE_HDF5 is False:
|
||||||
if USE_HDF5 and HAVE_HDF5:
|
return self.writeMa(fileName, **opts)
|
||||||
|
elif HAVE_HDF5 is True:
|
||||||
return self.writeHDF5(fileName, **opts)
|
return self.writeHDF5(fileName, **opts)
|
||||||
else:
|
else:
|
||||||
return self.writeMa(fileName, **opts)
|
raise Exception("h5py is required for writing .ma hdf5 files, but it could not be imported.")
|
||||||
|
|
||||||
def writeMeta(self, fileName):
|
def writeMeta(self, fileName):
|
||||||
"""Used to re-write meta info to the given file.
|
"""Used to re-write meta info to the given file.
|
||||||
@ -1050,7 +1046,6 @@ class MetaArray(object):
|
|||||||
self.writeHDF5Meta(f, 'info', self._info)
|
self.writeHDF5Meta(f, 'info', self._info)
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
def writeHDF5(self, fileName, **opts):
|
def writeHDF5(self, fileName, **opts):
|
||||||
## default options for writing datasets
|
## default options for writing datasets
|
||||||
comp = self.defaultCompression
|
comp = self.defaultCompression
|
||||||
@ -1088,7 +1083,6 @@ class MetaArray(object):
|
|||||||
if k in opts:
|
if k in opts:
|
||||||
dsOpts[k] = opts[k]
|
dsOpts[k] = opts[k]
|
||||||
|
|
||||||
|
|
||||||
## If mappable is in options, it disables chunking/compression
|
## If mappable is in options, it disables chunking/compression
|
||||||
if opts.get('mappable', False):
|
if opts.get('mappable', False):
|
||||||
dsOpts = {
|
dsOpts = {
|
||||||
|
@ -1,11 +1,15 @@
|
|||||||
import pyqtgraph as pg
|
import pyqtgraph as pg
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import sys
|
import sys
|
||||||
|
from copy import deepcopy
|
||||||
|
from collections import OrderedDict
|
||||||
from numpy.testing import assert_array_almost_equal, assert_almost_equal
|
from numpy.testing import assert_array_almost_equal, assert_almost_equal
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
np.random.seed(12345)
|
np.random.seed(12345)
|
||||||
|
|
||||||
|
|
||||||
def testSolve3D():
|
def testSolve3D():
|
||||||
p1 = np.array([[0,0,0,1],
|
p1 = np.array([[0,0,0,1],
|
||||||
[1,0,0,1],
|
[1,0,0,1],
|
||||||
@ -356,6 +360,29 @@ def test_eq():
|
|||||||
assert eq(a4, a4.copy())
|
assert eq(a4, a4.copy())
|
||||||
assert not eq(a4, a4.T)
|
assert not eq(a4, a4.T)
|
||||||
|
|
||||||
|
# test containers
|
||||||
|
|
||||||
|
assert not eq({'a': 1}, {'a': 1, 'b': 2})
|
||||||
|
assert not eq({'a': 1}, {'a': 2})
|
||||||
|
d1 = {'x': 1, 'y': np.nan, 3: ['a', np.nan, a3, 7, 2.3], 4: a4}
|
||||||
|
d2 = deepcopy(d1)
|
||||||
|
assert eq(d1, d2)
|
||||||
|
assert eq(OrderedDict(d1), OrderedDict(d2))
|
||||||
|
assert not eq(OrderedDict(d1), d2)
|
||||||
|
items = list(d1.items())
|
||||||
|
assert not eq(OrderedDict(items), OrderedDict(reversed(items)))
|
||||||
|
|
||||||
|
assert not eq([1,2,3], [1,2,3,4])
|
||||||
|
l1 = [d1, np.inf, -np.inf, np.nan]
|
||||||
|
l2 = deepcopy(l1)
|
||||||
|
t1 = tuple(l1)
|
||||||
|
t2 = tuple(l2)
|
||||||
|
assert eq(l1, l2)
|
||||||
|
assert eq(t1, t2)
|
||||||
|
|
||||||
|
assert eq(set(range(10)), set(range(10)))
|
||||||
|
assert not eq(set(range(10)), set(range(9)))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
test_interpolateArray()
|
test_interpolateArray()
|
Loading…
Reference in New Issue
Block a user