update h5py deps in metaarray
- update h5py usage to support latest version - bugfix in __getitem__ for fancy indexing - code cleanup
This commit is contained in:
Luke Campagnola
parent
268d25c125
commit
5e971b646f
@ -21,6 +21,13 @@ from ..python2_3 import basestring
|
||||
USE_HDF5 = True
|
||||
try:
|
||||
import h5py
|
||||
|
||||
# Older h5py versions tucked Group and Dataset deeper inside the library:
|
||||
if not hasattr(h5py, 'Group'):
|
||||
import h5py.highlevel
|
||||
h5py.Group = h5py.highlevel.Group
|
||||
h5py.Dataset = h5py.highlevel.Dataset
|
||||
|
||||
HAVE_HDF5 = True
|
||||
except:
|
||||
USE_HDF5 = False
|
||||
@ -124,7 +131,6 @@ class MetaArray(object):
|
||||
|
||||
def __init__(self, data=None, info=None, dtype=None, file=None, copy=False, **kwargs):
|
||||
object.__init__(self)
|
||||
#self._infoOwned = False
|
||||
self._isHDF = False
|
||||
|
||||
if file is not None:
|
||||
@ -190,58 +196,22 @@ class MetaArray(object):
|
||||
return ['MetaArray']
|
||||
else:
|
||||
return name == 'MetaArray'
|
||||
|
||||
#def __array_finalize__(self,obj):
|
||||
### array_finalize is called every time a MetaArray is created
|
||||
### (whereas __new__ is not necessarily called every time)
|
||||
|
||||
### obj is the object from which this array was generated (for example, when slicing or view()ing)
|
||||
|
||||
## We use the getattr method to set a default if 'obj' doesn't have the 'info' attribute
|
||||
##print "Create new MA from object", str(type(obj))
|
||||
##import traceback
|
||||
##traceback.print_stack()
|
||||
##print "finalize", type(self), type(obj)
|
||||
#if not hasattr(self, '_info'):
|
||||
##if isinstance(obj, MetaArray):
|
||||
##print " copy info:", obj._info
|
||||
#self._info = getattr(obj, '_info', [{}]*(obj.ndim+1))
|
||||
#self._infoOwned = False ## Do not make changes to _info until it is copied at least once
|
||||
##print " self info:", self._info
|
||||
|
||||
## We could have checked first whether self._info was already defined:
|
||||
##if not hasattr(self, 'info'):
|
||||
## self._info = getattr(obj, 'info', {})
|
||||
|
||||
|
||||
def __getitem__(self, ind):
|
||||
#print "getitem:", ind
|
||||
|
||||
## should catch scalar requests as early as possible to speed things up (?)
|
||||
|
||||
nInd = self._interpretIndexes(ind)
|
||||
|
||||
#a = np.ndarray.__getitem__(self, nInd)
|
||||
|
||||
a = self._data[nInd]
|
||||
if len(nInd) == self.ndim:
|
||||
if np.all([not isinstance(ind, slice) for ind in nInd]): ## no slices; we have requested a single value from the array
|
||||
if np.all([not isinstance(ind, (slice, np.ndarray)) for ind in nInd]): ## no slices; we have requested a single value from the array
|
||||
return a
|
||||
#if type(a) != type(self._data) and not isinstance(a, np.ndarray): ## indexing returned single value
|
||||
#return a
|
||||
|
||||
## indexing returned a sub-array; generate new info array to go with it
|
||||
#print " new MA:", type(a), a.shape
|
||||
info = []
|
||||
extraInfo = self._info[-1].copy()
|
||||
for i in range(0, len(nInd)): ## iterate over all axes
|
||||
#print " axis", i
|
||||
if type(nInd[i]) in [slice, list] or isinstance(nInd[i], np.ndarray): ## If the axis is sliced, keep the info but chop if necessary
|
||||
#print " slice axis", i, nInd[i]
|
||||
#a._info[i] = self._axisSlice(i, nInd[i])
|
||||
#print " info:", a._info[i]
|
||||
info.append(self._axisSlice(i, nInd[i]))
|
||||
else: ## If the axis is indexed, then move the information from that single index to the last info dictionary
|
||||
#print "indexed:", i, nInd[i], type(nInd[i])
|
||||
newInfo = self._axisSlice(i, nInd[i])
|
||||
name = None
|
||||
colName = None
|
||||
@ -270,16 +240,8 @@ class MetaArray(object):
|
||||
else:
|
||||
extraInfo['name'] = name
|
||||
|
||||
|
||||
#print "Lost info:", newInfo
|
||||
#a._info[i] = None
|
||||
#if 'name' in newInfo:
|
||||
#a._info[-1][newInfo['name']] = newInfo
|
||||
info.append(extraInfo)
|
||||
|
||||
#self._infoOwned = False
|
||||
#while None in a._info:
|
||||
#a._info.remove(None)
|
||||
return MetaArray(a, info=info)
|
||||
|
||||
@property
|
||||
@ -313,22 +275,15 @@ class MetaArray(object):
|
||||
return getattr(self._data, attr)
|
||||
else:
|
||||
raise AttributeError(attr)
|
||||
#return lambda *args, **kwargs: MetaArray(getattr(a.view(ndarray), attr)(*args, **kwargs)
|
||||
|
||||
def __eq__(self, b):
|
||||
return self._binop('__eq__', b)
|
||||
|
||||
def __ne__(self, b):
|
||||
return self._binop('__ne__', b)
|
||||
#if isinstance(b, MetaArray):
|
||||
#b = b.asarray()
|
||||
#return self.asarray() != b
|
||||
|
||||
def __sub__(self, b):
|
||||
return self._binop('__sub__', b)
|
||||
#if isinstance(b, MetaArray):
|
||||
#b = b.asarray()
|
||||
#return MetaArray(self.asarray() - b, info=self.infoCopy())
|
||||
|
||||
def __add__(self, b):
|
||||
return self._binop('__add__', b)
|
||||
@ -498,14 +453,7 @@ class MetaArray(object):
|
||||
numOk = True ## Named indices not started yet; numbered sill ok
|
||||
for i in range(0,len(ind)):
|
||||
(axis, index, isNamed) = self._interpretIndex(ind[i], i, numOk)
|
||||
#try:
|
||||
nInd[axis] = index
|
||||
#except:
|
||||
#print "ndim:", self.ndim
|
||||
#print "axis:", axis
|
||||
#print "index spec:", ind[i]
|
||||
#print "index num:", index
|
||||
#raise
|
||||
if isNamed:
|
||||
numOk = False
|
||||
return tuple(nInd)
|
||||
@ -684,9 +632,7 @@ class MetaArray(object):
|
||||
def __str__(self):
|
||||
return self.__repr__()
|
||||
|
||||
|
||||
def axisCollapsingFn(self, fn, axis=None, *args, **kargs):
|
||||
#arr = self.view(np.ndarray)
|
||||
fn = getattr(self._data, fn)
|
||||
if axis is None:
|
||||
return fn(axis, *args, **kargs)
|
||||
@ -823,8 +769,6 @@ class MetaArray(object):
|
||||
|
||||
## No axes are dynamic, just read the entire array in at once
|
||||
if dynAxis is None:
|
||||
#if rewriteDynamic is not None:
|
||||
#raise Exception("")
|
||||
if meta['type'] == 'object':
|
||||
if mmap:
|
||||
raise Exception('memmap not supported for arrays with dtype=object')
|
||||
@ -834,9 +778,7 @@ class MetaArray(object):
|
||||
subarr = np.memmap(fd, dtype=meta['type'], mode='r', shape=meta['shape'])
|
||||
else:
|
||||
subarr = np.fromstring(fd.read(), dtype=meta['type'])
|
||||
#subarr = subarr.view(subtype)
|
||||
subarr.shape = meta['shape']
|
||||
#subarr._info = meta['info']
|
||||
## One axis is dynamic, read in a frame at a time
|
||||
else:
|
||||
if mmap:
|
||||
@ -993,9 +935,9 @@ class MetaArray(object):
|
||||
data[k] = val
|
||||
for k in root:
|
||||
obj = root[k]
|
||||
if isinstance(obj, h5py.highlevel.Group):
|
||||
if isinstance(obj, h5py.Group):
|
||||
val = MetaArray.readHDF5Meta(obj)
|
||||
elif isinstance(obj, h5py.highlevel.Dataset):
|
||||
elif isinstance(obj, h5py.Dataset):
|
||||
if mmap:
|
||||
val = MetaArray.mapHDF5Array(obj)
|
||||
else:
|
||||
@ -1256,89 +1198,6 @@ class MetaArray(object):
|
||||
file.close()
|
||||
else:
|
||||
return ret
|
||||
|
||||
|
||||
|
||||
#class H5MetaList():
|
||||
|
||||
|
||||
#def rewriteContiguous(fileName, newName):
|
||||
#"""Rewrite a dynamic array file as contiguous"""
|
||||
#def _readData2(fd, meta, subtype, mmap):
|
||||
### read in axis values
|
||||
#dynAxis = None
|
||||
#frameSize = 1
|
||||
### read in axis values for any axis that specifies a length
|
||||
#for i in range(len(meta['info'])):
|
||||
#ax = meta['info'][i]
|
||||
#if ax.has_key('values_len'):
|
||||
#if ax['values_len'] == 'dynamic':
|
||||
#if dynAxis is not None:
|
||||
#raise Exception("MetaArray has more than one dynamic axis! (this is not allowed)")
|
||||
#dynAxis = i
|
||||
#else:
|
||||
#ax['values'] = fromstring(fd.read(ax['values_len']), dtype=ax['values_type'])
|
||||
#frameSize *= ax['values_len']
|
||||
#del ax['values_len']
|
||||
#del ax['values_type']
|
||||
|
||||
### No axes are dynamic, just read the entire array in at once
|
||||
#if dynAxis is None:
|
||||
#raise Exception('Array has no dynamic axes.')
|
||||
### One axis is dynamic, read in a frame at a time
|
||||
#else:
|
||||
#if mmap:
|
||||
#raise Exception('memmap not supported for non-contiguous arrays. Use rewriteContiguous() to convert.')
|
||||
#ax = meta['info'][dynAxis]
|
||||
#xVals = []
|
||||
#frames = []
|
||||
#frameShape = list(meta['shape'])
|
||||
#frameShape[dynAxis] = 1
|
||||
#frameSize = np.prod(frameShape)
|
||||
#n = 0
|
||||
#while True:
|
||||
### Extract one non-blank line
|
||||
#while True:
|
||||
#line = fd.readline()
|
||||
#if line != '\n':
|
||||
#break
|
||||
#if line == '':
|
||||
#break
|
||||
|
||||
### evaluate line
|
||||
#inf = eval(line)
|
||||
|
||||
### read data block
|
||||
##print "read %d bytes as %s" % (inf['len'], meta['type'])
|
||||
#if meta['type'] == 'object':
|
||||
#data = pickle.loads(fd.read(inf['len']))
|
||||
#else:
|
||||
#data = fromstring(fd.read(inf['len']), dtype=meta['type'])
|
||||
|
||||
#if data.size != frameSize * inf['numFrames']:
|
||||
##print data.size, frameSize, inf['numFrames']
|
||||
#raise Exception("Wrong frame size in MetaArray file! (frame %d)" % n)
|
||||
|
||||
### read in data block
|
||||
#shape = list(frameShape)
|
||||
#shape[dynAxis] = inf['numFrames']
|
||||
#data.shape = shape
|
||||
#frames.append(data)
|
||||
|
||||
#n += inf['numFrames']
|
||||
#if 'xVals' in inf:
|
||||
#xVals.extend(inf['xVals'])
|
||||
#subarr = np.concatenate(frames, axis=dynAxis)
|
||||
#if len(xVals)> 0:
|
||||
#ax['values'] = array(xVals, dtype=ax['values_type'])
|
||||
#del ax['values_len']
|
||||
#del ax['values_type']
|
||||
#subarr = subarr.view(subtype)
|
||||
#subarr._info = meta['info']
|
||||
#return subarr
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
Reference in New Issue
Block a user