don't modify info from v1 files, move info correction to hdf reading
This commit is contained in:
parent
ae61d3582e
commit
c484c86417
@ -14,20 +14,18 @@ import types, copy, threading, os, re
|
|||||||
import pickle
|
import pickle
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from ..python2_3 import basestring
|
from ..python2_3 import basestring
|
||||||
#import traceback
|
|
||||||
|
|
||||||
## By default, the library will use HDF5 when writing files.
|
## By default, the library will use HDF5 when writing files.
|
||||||
## This can be overridden by setting USE_HDF5 = False
|
## This can be overridden by setting USE_HDF5 = False
|
||||||
USE_HDF5 = True
|
USE_HDF5 = True
|
||||||
try:
|
try:
|
||||||
import h5py
|
import h5py.highlevel
|
||||||
HAVE_HDF5 = True
|
HAVE_HDF5 = True
|
||||||
except:
|
except:
|
||||||
USE_HDF5 = False
|
USE_HDF5 = False
|
||||||
HAVE_HDF5 = False
|
HAVE_HDF5 = False
|
||||||
|
|
||||||
if HAVE_HDF5:
|
|
||||||
import h5py.highlevel
|
|
||||||
|
|
||||||
def axis(name=None, cols=None, values=None, units=None):
|
def axis(name=None, cols=None, values=None, units=None):
|
||||||
"""Convenience function for generating axis descriptions when defining MetaArrays"""
|
"""Convenience function for generating axis descriptions when defining MetaArrays"""
|
||||||
@ -778,20 +776,6 @@ class MetaArray(object):
|
|||||||
#print ret
|
#print ret
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
def fix_info(self, info):
|
|
||||||
"""
|
|
||||||
Recursive version
|
|
||||||
"""
|
|
||||||
if isinstance(info, list):
|
|
||||||
for i in range(len(info)):
|
|
||||||
info[i] = self.fix_info(info[i])
|
|
||||||
elif isinstance(info, dict):
|
|
||||||
for k in info.keys():
|
|
||||||
info[k] = self.fix_info(info[k])
|
|
||||||
elif isinstance(info, bytes): # change all bytestrings to string and remove internal quotes
|
|
||||||
info = info.decode('utf-8').replace("\'", '')
|
|
||||||
return info
|
|
||||||
|
|
||||||
def _readData1(self, fd, meta, mmap=False, **kwds):
|
def _readData1(self, fd, meta, mmap=False, **kwds):
|
||||||
## Read array data from the file descriptor for MetaArray v1 files
|
## Read array data from the file descriptor for MetaArray v1 files
|
||||||
## read in axis values for any axis that specifies a length
|
## read in axis values for any axis that specifies a length
|
||||||
@ -802,7 +786,7 @@ class MetaArray(object):
|
|||||||
frameSize *= ax['values_len']
|
frameSize *= ax['values_len']
|
||||||
del ax['values_len']
|
del ax['values_len']
|
||||||
del ax['values_type']
|
del ax['values_type']
|
||||||
self._info = self.fix_info(meta['info'])
|
self._info = meta['info']
|
||||||
if not kwds.get("readAllData", True):
|
if not kwds.get("readAllData", True):
|
||||||
return
|
return
|
||||||
## the remaining data is the actual array
|
## the remaining data is the actual array
|
||||||
@ -830,7 +814,7 @@ class MetaArray(object):
|
|||||||
frameSize *= ax['values_len']
|
frameSize *= ax['values_len']
|
||||||
del ax['values_len']
|
del ax['values_len']
|
||||||
del ax['values_type']
|
del ax['values_type']
|
||||||
self._info = self.fix_info(meta['info'])
|
self._info = meta['info']
|
||||||
if not kwds.get("readAllData", True):
|
if not kwds.get("readAllData", True):
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -901,10 +885,8 @@ class MetaArray(object):
|
|||||||
newSubset = list(subset[:])
|
newSubset = list(subset[:])
|
||||||
newSubset[dynAxis] = slice(dStart, dStop)
|
newSubset[dynAxis] = slice(dStart, dStop)
|
||||||
if dStop > dStart:
|
if dStop > dStart:
|
||||||
#print n, data.shape, " => ", newSubset, data[tuple(newSubset)].shape
|
|
||||||
frames.append(data[tuple(newSubset)].copy())
|
frames.append(data[tuple(newSubset)].copy())
|
||||||
else:
|
else:
|
||||||
#data = data[subset].copy() ## what's this for??
|
|
||||||
frames.append(data)
|
frames.append(data)
|
||||||
|
|
||||||
n += inf['numFrames']
|
n += inf['numFrames']
|
||||||
@ -915,12 +897,8 @@ class MetaArray(object):
|
|||||||
ax['values'] = np.array(xVals, dtype=ax['values_type'])
|
ax['values'] = np.array(xVals, dtype=ax['values_type'])
|
||||||
del ax['values_len']
|
del ax['values_len']
|
||||||
del ax['values_type']
|
del ax['values_type']
|
||||||
#subarr = subarr.view(subtype)
|
self._info = meta['info']
|
||||||
#subarr._info = meta['info']
|
|
||||||
self._info = self.fix_info(meta['info'])
|
|
||||||
self._data = subarr
|
self._data = subarr
|
||||||
#raise Exception() ## stress-testing
|
|
||||||
#return subarr
|
|
||||||
|
|
||||||
def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
|
def _readHDF5(self, fileName, readAllData=None, writable=False, **kargs):
|
||||||
if 'close' in kargs and readAllData is None: ## for backward compatibility
|
if 'close' in kargs and readAllData is None: ## for backward compatibility
|
||||||
@ -957,7 +935,7 @@ class MetaArray(object):
|
|||||||
if ver > MetaArray.version:
|
if ver > MetaArray.version:
|
||||||
print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
|
print("Warning: This file was written with MetaArray version %s, but you are using version %s. (Will attempt to read anyway)" % (str(ver), str(MetaArray.version)))
|
||||||
meta = MetaArray.readHDF5Meta(f['info'])
|
meta = MetaArray.readHDF5Meta(f['info'])
|
||||||
self._info = self.fix_info(meta)
|
self._info = meta
|
||||||
|
|
||||||
if writable or not readAllData: ## read all data, convert to ndarray, close file
|
if writable or not readAllData: ## read all data, convert to ndarray, close file
|
||||||
self._data = f['data']
|
self._data = f['data']
|
||||||
@ -982,12 +960,7 @@ class MetaArray(object):
|
|||||||
MetaArray._h5py_metaarray = proc._import('pyqtgraph.metaarray')
|
MetaArray._h5py_metaarray = proc._import('pyqtgraph.metaarray')
|
||||||
ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
|
ma = MetaArray._h5py_metaarray.MetaArray(file=fileName)
|
||||||
self._data = ma.asarray()._getValue()
|
self._data = ma.asarray()._getValue()
|
||||||
self._info = self.fix_info(ma._info._getValue())
|
self._info = ma._info._getValue()
|
||||||
#print MetaArray._hdf5Process
|
|
||||||
#import inspect
|
|
||||||
#print MetaArray, id(MetaArray), inspect.getmodule(MetaArray)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def mapHDF5Array(data, writable=False):
|
def mapHDF5Array(data, writable=False):
|
||||||
@ -1000,9 +973,6 @@ class MetaArray(object):
|
|||||||
raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
|
raise Exception("This dataset uses chunked storage; it can not be memory-mapped. (store using mappable=True)")
|
||||||
return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
|
return np.memmap(filename=data.file.filename, offset=off, dtype=data.dtype, shape=data.shape, mode=mode)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def readHDF5Meta(root, mmap=False):
|
def readHDF5Meta(root, mmap=False):
|
||||||
data = {}
|
data = {}
|
||||||
@ -1010,6 +980,8 @@ class MetaArray(object):
|
|||||||
## Pull list of values from attributes and child objects
|
## Pull list of values from attributes and child objects
|
||||||
for k in root.attrs:
|
for k in root.attrs:
|
||||||
val = root.attrs[k]
|
val = root.attrs[k]
|
||||||
|
if isinstance(val, bytes):
|
||||||
|
val = val.decode()
|
||||||
if isinstance(val, basestring): ## strings need to be re-evaluated to their original types
|
if isinstance(val, basestring): ## strings need to be re-evaluated to their original types
|
||||||
try:
|
try:
|
||||||
val = eval(val)
|
val = eval(val)
|
||||||
@ -1048,8 +1020,7 @@ class MetaArray(object):
|
|||||||
else:
|
else:
|
||||||
raise Exception("Don't understand metaType '%s'" % typ)
|
raise Exception("Don't understand metaType '%s'" % typ)
|
||||||
|
|
||||||
|
def write(self, fileName, version=2, **opts):
|
||||||
def write(self, fileName, **opts):
|
|
||||||
"""Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
|
"""Write this object to a file. The object can be restored by calling MetaArray(file=fileName)
|
||||||
opts:
|
opts:
|
||||||
appendAxis: the name (or index) of the appendable axis. Allows the array to grow.
|
appendAxis: the name (or index) of the appendable axis. Allows the array to grow.
|
||||||
@ -1057,11 +1028,15 @@ class MetaArray(object):
|
|||||||
compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
|
compression: None, 'gzip' (good compression), 'lzf' (fast compression), etc.
|
||||||
chunks: bool or tuple specifying chunk shape
|
chunks: bool or tuple specifying chunk shape
|
||||||
"""
|
"""
|
||||||
|
if version == 1:
|
||||||
if USE_HDF5 and HAVE_HDF5:
|
return self.writeMa(fileName, **opts)
|
||||||
|
elif USE_HDF5 and HAVE_HDF5:
|
||||||
return self.writeHDF5(fileName, **opts)
|
return self.writeHDF5(fileName, **opts)
|
||||||
else:
|
else:
|
||||||
return self.writeMa(fileName, **opts)
|
if not HAVE_HDF5:
|
||||||
|
raise Exception("h5py is required for writing .ma version 2 files")
|
||||||
|
else:
|
||||||
|
raise Exception("HDF5 is required for writing .ma version 2 files, but it has been disabled.")
|
||||||
|
|
||||||
def writeMeta(self, fileName):
|
def writeMeta(self, fileName):
|
||||||
"""Used to re-write meta info to the given file.
|
"""Used to re-write meta info to the given file.
|
||||||
@ -1074,7 +1049,6 @@ class MetaArray(object):
|
|||||||
self.writeHDF5Meta(f, 'info', self._info)
|
self.writeHDF5Meta(f, 'info', self._info)
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
def writeHDF5(self, fileName, **opts):
|
def writeHDF5(self, fileName, **opts):
|
||||||
## default options for writing datasets
|
## default options for writing datasets
|
||||||
comp = self.defaultCompression
|
comp = self.defaultCompression
|
||||||
@ -1112,7 +1086,6 @@ class MetaArray(object):
|
|||||||
if k in opts:
|
if k in opts:
|
||||||
dsOpts[k] = opts[k]
|
dsOpts[k] = opts[k]
|
||||||
|
|
||||||
|
|
||||||
## If mappable is in options, it disables chunking/compression
|
## If mappable is in options, it disables chunking/compression
|
||||||
if opts.get('mappable', False):
|
if opts.get('mappable', False):
|
||||||
dsOpts = {
|
dsOpts = {
|
||||||
|
Loading…
Reference in New Issue
Block a user