From e234d90f027975d6391a3e6d913042a0023c2397 Mon Sep 17 00:00:00 2001 From: Luke Campagnola <> Date: Sat, 12 Jan 2013 14:31:49 -0500 Subject: [PATCH] Bugfixes: - GraphicsItem.pixelVectors copies cached results before returning - Multiprocess fixes for Windows: - mmap/shm uses anonymous maps rather than tempfiles - avoid use of getppid and setpgrp - work around hmac authentication bug (use os.urandom to generate key) --- __init__.py | 4 ++- functions.py | 9 ++++-- graphicsItems/GraphicsItem.py | 6 ++-- graphicsItems/ROI.py | 3 +- multiprocess/bootstrap.py | 8 +++-- multiprocess/processes.py | 23 ++++++++------ widgets/RemoteGraphicsView.py | 59 +++++++++++++++++++++++------------ 7 files changed, 71 insertions(+), 41 deletions(-) diff --git a/__init__.py b/__init__.py index 93d9f7b8..c35992a2 100644 --- a/__init__.py +++ b/__init__.py @@ -4,7 +4,7 @@ PyQtGraph - Scientific Graphics and GUI Library for Python www.pyqtgraph.org """ -__version__ = None +__version__ = '0.9.5' ### import all the goodies and add some helper functions for easy CLI use @@ -49,6 +49,8 @@ CONFIG_OPTIONS = { 'background': (0, 0, 0), ## default background for GraphicsWidget 'antialias': False, 'editorCommand': None, ## command used to invoke code editor from ConsoleWidgets + 'useWeave': True, ## Use weave to speed up some operations, if it is available + 'weaveDebug': False, ## Print full error message if weave compile fails } diff --git a/functions.py b/functions.py index 23b2580c..00107927 100644 --- a/functions.py +++ b/functions.py @@ -23,6 +23,7 @@ SI_PREFIXES_ASCII = 'yzafpnum kMGTPEZY' from .Qt import QtGui, QtCore, USE_PYSIDE +from pyqtgraph import getConfigOption import numpy as np import decimal, re import ctypes @@ -30,9 +31,10 @@ import ctypes try: import scipy.ndimage HAVE_SCIPY = True + WEAVE_DEBUG = getConfigOption('weaveDebug') try: import scipy.weave - USE_WEAVE = True + USE_WEAVE = getConfigOption('useWeave') except: USE_WEAVE = False except ImportError: @@ -631,7 +633,8 @@ def rescaleData(data, scale, offset, dtype=None): data = newData.reshape(data.shape) except: if USE_WEAVE: - debug.printExc("Error; disabling weave.") + if WEAVE_DEBUG: + debug.printExc("Error; disabling weave.") USE_WEAVE = False #p = np.poly1d([scale, -offset*scale]) @@ -1871,4 +1874,4 @@ def pseudoScatter(data, spacing=None, shuffle=True): yvals[i] = y - return yvals[np.argsort(inds)] ## un-shuffle values before returning \ No newline at end of file + return yvals[np.argsort(inds)] ## un-shuffle values before returning diff --git a/graphicsItems/GraphicsItem.py b/graphicsItems/GraphicsItem.py index 2314709d..c90821a3 100644 --- a/graphicsItems/GraphicsItem.py +++ b/graphicsItems/GraphicsItem.py @@ -197,14 +197,14 @@ class GraphicsItem(object): ## check local cache if direction is None and dt == self._pixelVectorCache[0]: - return self._pixelVectorCache[1] + return map(Point, self._pixelVectorCache[1]) ## return a *copy* ## check global cache key = (dt.m11(), dt.m21(), dt.m31(), dt.m12(), dt.m22(), dt.m32(), dt.m31(), dt.m32()) pv = self._pixelVectorGlobalCache.get(key, None) if pv is not None: self._pixelVectorCache = [dt, pv] - return pv + return map(Point,pv) ## return a *copy* if direction is None: @@ -547,4 +547,4 @@ class GraphicsItem(object): #def update(self): #self._qtBaseClass.update(self) - #print "Update:", self \ No newline at end of file + #print "Update:", self diff --git a/graphicsItems/ROI.py b/graphicsItems/ROI.py index e3f094ff..c3620edb 100644 --- a/graphicsItems/ROI.py +++ b/graphicsItems/ROI.py @@ -1783,8 +1783,7 @@ class LineSegmentROI(ROI): dh = h2-h1 if dh.length() == 0: return p - pxv = self.pixelVectors(h2-h1)[1] - + pxv = self.pixelVectors(dh)[1] if pxv is None: return p diff --git a/multiprocess/bootstrap.py b/multiprocess/bootstrap.py index e0d1c02c..6ac9fce4 100644 --- a/multiprocess/bootstrap.py +++ b/multiprocess/bootstrap.py @@ -2,8 +2,10 @@ import sys, pickle, os if __name__ == '__main__': - os.setpgrp() ## prevents signals (notably keyboard interrupt) being forwarded from parent to this process - name, port, authkey, targetStr, path = pickle.load(sys.stdin) + if hasattr(os, 'setpgrp'): + os.setpgrp() ## prevents signals (notably keyboard interrupt) being forwarded from parent to this process + name, port, authkey, ppid, targetStr, path = pickle.load(sys.stdin) + #print "key:", ' '.join([str(ord(x)) for x in authkey]) if path is not None: ## rewrite sys.path without assigning a new object--no idea who already has a reference to the existing list. while len(sys.path) > 0: @@ -12,5 +14,5 @@ if __name__ == '__main__': #import pyqtgraph #import pyqtgraph.multiprocess.processes target = pickle.loads(targetStr) ## unpickling the target should import everything we need - target(name, port, authkey) + target(name, port, authkey, ppid) sys.exit(0) diff --git a/multiprocess/processes.py b/multiprocess/processes.py index f95a3ec4..1322e78e 100644 --- a/multiprocess/processes.py +++ b/multiprocess/processes.py @@ -54,12 +54,13 @@ class Process(RemoteEventHandler): executable = sys.executable ## random authentication key - authkey = ''.join([chr(random.getrandbits(7)) for i in range(20)]) - + authkey = os.urandom(20) + #print "key:", ' '.join([str(ord(x)) for x in authkey]) ## Listen for connection from remote process (and find free port number) port = 10000 while True: try: + ## hmac authentication appears to be broken on windows (says AuthenticationError: digest received was wrong) l = multiprocessing.connection.Listener(('localhost', int(port)), authkey=authkey) break except socket.error as ex: @@ -73,7 +74,8 @@ class Process(RemoteEventHandler): self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE) targetStr = pickle.dumps(target) ## double-pickle target so that child has a chance to ## set its sys.path properly before unpickling the target - pickle.dump((name+'_child', port, authkey, targetStr, sysPath), self.proc.stdin) + pid = os.getpid() # we must sent pid to child because windows does not have getppid + pickle.dump((name+'_child', port, authkey, pid, targetStr, sysPath), self.proc.stdin) self.proc.stdin.close() ## open connection for remote process @@ -92,10 +94,11 @@ class Process(RemoteEventHandler): time.sleep(0.05) -def startEventLoop(name, port, authkey): +def startEventLoop(name, port, authkey, ppid): conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey) global HANDLER - HANDLER = RemoteEventHandler(conn, name, os.getppid()) + #ppid = 0 if not hasattr(os, 'getppid') else os.getppid() + HANDLER = RemoteEventHandler(conn, name, ppid) while True: try: HANDLER.processRequests() # exception raised when the loop should exit @@ -161,6 +164,7 @@ class ForkedProcess(RemoteEventHandler): proxyId = LocalObjectProxy.registerObject(v) proxyIDs[k] = proxyId + ppid = os.getpid() # write this down now; windows doesn't have getppid pid = os.fork() if pid == 0: self.isParent = False @@ -200,9 +204,9 @@ class ForkedProcess(RemoteEventHandler): if 'random' in sys.modules: sys.modules['random'].seed(os.getpid() ^ int(time.time()*10000%10000)) - RemoteEventHandler.__init__(self, remoteConn, name+'_child', pid=os.getppid()) + #ppid = 0 if not hasattr(os, 'getppid') else os.getppid() + RemoteEventHandler.__init__(self, remoteConn, name+'_child', pid=ppid) - ppid = os.getppid() self.forkedProxies = {} for name, proxyId in proxyIDs.iteritems(): self.forkedProxies[name] = ObjectProxy(ppid, proxyId=proxyId, typeStr=repr(preProxy[name])) @@ -318,7 +322,7 @@ class QtProcess(Process): except ClosedError: self.timer.stop() -def startQtEventLoop(name, port, authkey): +def startQtEventLoop(name, port, authkey, ppid): conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey) from pyqtgraph.Qt import QtGui, QtCore #from PyQt4 import QtGui, QtCore @@ -330,7 +334,8 @@ def startQtEventLoop(name, port, authkey): ## until it is explicitly closed by the parent process. global HANDLER - HANDLER = RemoteQtEventHandler(conn, name, os.getppid()) + #ppid = 0 if not hasattr(os, 'getppid') else os.getppid() + HANDLER = RemoteQtEventHandler(conn, name, ppid) HANDLER.startEventTimer() app.exec_() diff --git a/widgets/RemoteGraphicsView.py b/widgets/RemoteGraphicsView.py index d8e720b5..bea5a02d 100644 --- a/widgets/RemoteGraphicsView.py +++ b/widgets/RemoteGraphicsView.py @@ -3,7 +3,7 @@ import pyqtgraph.multiprocess as mp import pyqtgraph as pg from .GraphicsView import GraphicsView import numpy as np -import mmap, tempfile, ctypes, atexit +import mmap, tempfile, ctypes, atexit, sys, random __all__ = ['RemoteGraphicsView'] @@ -30,10 +30,12 @@ class RemoteGraphicsView(QtGui.QWidget): self.setFocusPolicy(self._view.focusPolicy()) self.setSizePolicy(QtGui.QSizePolicy.Expanding, QtGui.QSizePolicy.Expanding) self.setMouseTracking(True) - + self.shm = None shmFileName = self._view.shmFileName() - self.shmFile = open(shmFileName, 'r') - self.shm = mmap.mmap(self.shmFile.fileno(), mmap.PAGESIZE, mmap.MAP_SHARED, mmap.PROT_READ) + if 'win' in sys.platform: + self.shmtag = shmFileName + else: + self.shmFile = open(shmFileName, 'r') self._view.sceneRendered.connect(mp.proxy(self.remoteSceneChanged)) #, callSync='off')) ## Note: we need synchronous signals @@ -53,11 +55,16 @@ class RemoteGraphicsView(QtGui.QWidget): return QtCore.QSize(*self._sizeHint) def remoteSceneChanged(self, data): - w, h, size = data + w, h, size, newfile = data #self._sizeHint = (whint, hhint) - if self.shm.size != size: - self.shm.close() - self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ) + if self.shm is None or self.shm.size != size: + if self.shm is not None: + self.shm.close() + if 'win' in sys.platform: + self.shmtag = newfile ## on windows, we create a new tag for every resize + self.shm = mmap.mmap(-1, size, self.shmtag) ## can't use tmpfile on windows because the file can only be opened once. + else: + self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ) self.shm.seek(0) self._img = QtGui.QImage(self.shm.read(w*h*4), w, h, QtGui.QImage.Format_ARGB32) self.update() @@ -112,13 +119,14 @@ class Renderer(GraphicsView): def __init__(self, *args, **kwds): ## Create shared memory for rendered image - #fd = os.open('/tmp/mmaptest', os.O_CREAT | os.O_TRUNC | os.O_RDWR) - #os.write(fd, '\x00' * mmap.PAGESIZE) - self.shmFile = tempfile.NamedTemporaryFile(prefix='pyqtgraph_shmem_') - self.shmFile.write('\x00' * mmap.PAGESIZE) - #fh.flush() - fd = self.shmFile.fileno() - self.shm = mmap.mmap(fd, mmap.PAGESIZE, mmap.MAP_SHARED, mmap.PROT_WRITE) + if 'win' in sys.platform: + self.shmtag = "pyqtgraph_shmem_" + ''.join([chr((random.getrandbits(20)%25) + 97) for i in range(20)]) + self.shm = mmap.mmap(-1, mmap.PAGESIZE, self.shmtag) # use anonymous mmap on windows + else: + self.shmFile = tempfile.NamedTemporaryFile(prefix='pyqtgraph_shmem_') + self.shmFile.write('\x00' * mmap.PAGESIZE) + fd = self.shmFile.fileno() + self.shm = mmap.mmap(fd, mmap.PAGESIZE, mmap.MAP_SHARED, mmap.PROT_WRITE) atexit.register(self.close) GraphicsView.__init__(self, *args, **kwds) @@ -130,10 +138,14 @@ class Renderer(GraphicsView): def close(self): self.shm.close() - self.shmFile.close() + if 'win' not in sys.platform: + self.shmFile.close() def shmFileName(self): - return self.shmFile.name + if 'win' in sys.platform: + return self.shmtag + else: + return self.shmFile.name def update(self): self.img = None @@ -152,7 +164,14 @@ class Renderer(GraphicsView): return size = self.width() * self.height() * 4 if size > self.shm.size(): - self.shm.resize(size) + if 'win' in sys.platform: + ## windows says "WindowsError: [Error 87] the parameter is incorrect" if we try to resize the mmap + self.shm.close() + ## it also says (sometimes) 'access is denied' if we try to reuse the tag. + self.shmtag = "pyqtgraph_shmem_" + ''.join([chr((random.getrandbits(20)%25) + 97) for i in range(20)]) + self.shm = mmap.mmap(-1, size, self.shmtag) + else: + self.shm.resize(size) address = ctypes.addressof(ctypes.c_char.from_buffer(self.shm, 0)) ## render the scene directly to shared memory @@ -161,7 +180,7 @@ class Renderer(GraphicsView): p = QtGui.QPainter(self.img) self.render(p, self.viewRect(), self.rect()) p.end() - self.sceneRendered.emit((self.width(), self.height(), self.shm.size())) + self.sceneRendered.emit((self.width(), self.height(), self.shm.size(), self.shmFileName())) def mousePressEvent(self, typ, pos, gpos, btn, btns, mods): typ = QtCore.QEvent.Type(typ) @@ -202,4 +221,4 @@ class Renderer(GraphicsView): - \ No newline at end of file +