From 89c04c8a8128ee73f55c575d79afc0bcecc85bda Mon Sep 17 00:00:00 2001 From: Luke Campagnola Date: Fri, 14 Mar 2014 16:23:23 -0400 Subject: [PATCH] Corrected bug in multiprocess causing deadlock at exit Multiprocess debugging messages now use one color per process Corrected RemoteGraphicsView not setting correct pg options on remote process New debugging tools: * util.cprint for printing color on terminal (based on colorama) * debug.ThreadColor causes each thread to print in a different color * debug.PeriodicTrace used for debugging deadlocks * Mutex for detecting deadlocks --- pyqtgraph/debug.py | 76 +++++- pyqtgraph/multiprocess/processes.py | 55 +++-- pyqtgraph/multiprocess/remoteproxy.py | 5 +- pyqtgraph/util/Mutex.py | 277 +++++++++++++++++++++ pyqtgraph/util/colorama/LICENSE.txt | 28 +++ pyqtgraph/util/colorama/README.txt | 304 ++++++++++++++++++++++++ pyqtgraph/util/colorama/__init__.py | 0 pyqtgraph/util/colorama/win32.py | 134 +++++++++++ pyqtgraph/util/colorama/winterm.py | 120 ++++++++++ pyqtgraph/util/cprint.py | 101 ++++++++ pyqtgraph/widgets/RemoteGraphicsView.py | 3 +- 11 files changed, 1079 insertions(+), 24 deletions(-) create mode 100644 pyqtgraph/util/Mutex.py create mode 100644 pyqtgraph/util/colorama/LICENSE.txt create mode 100644 pyqtgraph/util/colorama/README.txt create mode 100644 pyqtgraph/util/colorama/__init__.py create mode 100644 pyqtgraph/util/colorama/win32.py create mode 100644 pyqtgraph/util/colorama/winterm.py create mode 100644 pyqtgraph/util/cprint.py diff --git a/pyqtgraph/debug.py b/pyqtgraph/debug.py index f208f3a5..4756423c 100644 --- a/pyqtgraph/debug.py +++ b/pyqtgraph/debug.py @@ -7,10 +7,12 @@ Distributed under MIT/X11 license. See license.txt for more infomation. from __future__ import print_function -import sys, traceback, time, gc, re, types, weakref, inspect, os, cProfile +import sys, traceback, time, gc, re, types, weakref, inspect, os, cProfile, threading from . import ptime from numpy import ndarray from .Qt import QtCore, QtGui +from .util.Mutex import Mutex +from .util import cprint __ftraceDepth = 0 def ftrace(func): @@ -991,3 +993,75 @@ class PrintDetector(object): def flush(self): self.stdout.flush() + + +class PeriodicTrace(object): + """ + Used to debug freezing by starting a new thread that reports on the + location of the main thread periodically. + """ + class ReportThread(QtCore.QThread): + def __init__(self): + self.frame = None + self.ind = 0 + self.lastInd = None + self.lock = Mutex() + QtCore.QThread.__init__(self) + + def notify(self, frame): + with self.lock: + self.frame = frame + self.ind += 1 + + def run(self): + while True: + time.sleep(1) + with self.lock: + if self.lastInd != self.ind: + print("== Trace %d: ==" % self.ind) + traceback.print_stack(self.frame) + self.lastInd = self.ind + + def __init__(self): + self.mainThread = threading.current_thread() + self.thread = PeriodicTrace.ReportThread() + self.thread.start() + sys.settrace(self.trace) + + def trace(self, frame, event, arg): + if threading.current_thread() is self.mainThread: # and 'threading' not in frame.f_code.co_filename: + self.thread.notify(frame) + # print("== Trace ==", event, arg) + # traceback.print_stack(frame) + return self.trace + + + +class ThreadColor(object): + """ + Wrapper on stdout/stderr that colors text by the current thread ID. + + *stream* must be 'stdout' or 'stderr'. + """ + colors = {} + lock = Mutex() + + def __init__(self, stream): + self.stream = getattr(sys, stream) + self.err = stream == 'stderr' + setattr(sys, stream, self) + + def write(self, msg): + with self.lock: + cprint.cprint(self.stream, self.color(), msg, -1, stderr=self.err) + + def flush(self): + with self.lock: + self.stream.flush() + + def color(self): + tid = threading.current_thread() + if tid not in self.colors: + c = (len(self.colors) % 15) + 1 + self.colors[tid] = c + return self.colors[tid] diff --git a/pyqtgraph/multiprocess/processes.py b/pyqtgraph/multiprocess/processes.py index fac985e9..0dfb80b9 100644 --- a/pyqtgraph/multiprocess/processes.py +++ b/pyqtgraph/multiprocess/processes.py @@ -1,13 +1,15 @@ -from .remoteproxy import RemoteEventHandler, ClosedError, NoResultError, LocalObjectProxy, ObjectProxy import subprocess, atexit, os, sys, time, random, socket, signal import multiprocessing.connection -from ..Qt import USE_PYSIDE - try: import cPickle as pickle except ImportError: import pickle +from .remoteproxy import RemoteEventHandler, ClosedError, NoResultError, LocalObjectProxy, ObjectProxy +from ..Qt import USE_PYSIDE +from ..util import cprint # color printing for debugging + + __all__ = ['Process', 'QtProcess', 'ForkedProcess', 'ClosedError', 'NoResultError'] class Process(RemoteEventHandler): @@ -35,7 +37,8 @@ class Process(RemoteEventHandler): return objects either by proxy or by value (if they are picklable). See ProxyObject for more information. """ - + _process_count = 1 # just used for assigning colors to each process for debugging + def __init__(self, name=None, target=None, executable=None, copySysPath=True, debug=False, timeout=20, wrapStdout=None): """ ============== ============================================================= @@ -64,7 +67,7 @@ class Process(RemoteEventHandler): name = str(self) if executable is None: executable = sys.executable - self.debug = debug + self.debug = 7 if debug is True else False # 7 causes printing in white ## random authentication key authkey = os.urandom(20) @@ -82,6 +85,13 @@ class Process(RemoteEventHandler): sysPath = sys.path if copySysPath else None bootstrap = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bootstrap.py')) self.debugMsg('Starting child process (%s %s)' % (executable, bootstrap)) + + # Decide on printing color for this process + if debug: + procDebug = (Process._process_count%6) + 1 # pick a color for this process to print in + Process._process_count += 1 + else: + procDebug = False if wrapStdout is None: wrapStdout = sys.platform.startswith('win') @@ -94,8 +104,8 @@ class Process(RemoteEventHandler): self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE, stdout=stdout, stderr=stderr) ## to circumvent the bug and still make the output visible, we use ## background threads to pass data from pipes to stdout/stderr - self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout") - self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr") + self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout", procDebug) + self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr", procDebug) else: self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE) @@ -112,7 +122,7 @@ class Process(RemoteEventHandler): targetStr=targetStr, path=sysPath, pyside=USE_PYSIDE, - debug=debug + debug=procDebug ) pickle.dump(data, self.proc.stdin) self.proc.stdin.close() @@ -128,8 +138,8 @@ class Process(RemoteEventHandler): continue else: raise - - RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=debug) + + RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=self.debug) self.debugMsg('Connected to child process.') atexit.register(self.join) @@ -159,10 +169,11 @@ class Process(RemoteEventHandler): def startEventLoop(name, port, authkey, ppid, debug=False): if debug: import os - print('[%d] connecting to server at port localhost:%d, authkey=%s..' % (os.getpid(), port, repr(authkey))) + cprint.cout(debug, '[%d] connecting to server at port localhost:%d, authkey=%s..\n' + % (os.getpid(), port, repr(authkey)), -1) conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey) if debug: - print('[%d] connected; starting remote proxy.' % os.getpid()) + cprint.cout(debug, '[%d] connected; starting remote proxy.\n' % os.getpid(), -1) global HANDLER #ppid = 0 if not hasattr(os, 'getppid') else os.getppid() HANDLER = RemoteEventHandler(conn, name, ppid, debug=debug) @@ -372,17 +383,17 @@ class QtProcess(Process): def __init__(self, **kwds): if 'target' not in kwds: kwds['target'] = startQtEventLoop + from ..Qt import QtGui ## avoid module-level import to keep bootstrap snappy. self._processRequests = kwds.pop('processRequests', True) + if self._processRequests and QtGui.QApplication.instance() is None: + raise Exception("Must create QApplication before starting QtProcess, or use QtProcess(processRequests=False)") Process.__init__(self, **kwds) self.startEventTimer() def startEventTimer(self): - from ..Qt import QtGui, QtCore ## avoid module-level import to keep bootstrap snappy. + from ..Qt import QtCore ## avoid module-level import to keep bootstrap snappy. self.timer = QtCore.QTimer() if self._processRequests: - app = QtGui.QApplication.instance() - if app is None: - raise Exception("Must create QApplication before starting QtProcess, or use QtProcess(processRequests=False)") self.startRequestProcessing() def startRequestProcessing(self, interval=0.01): @@ -404,10 +415,10 @@ class QtProcess(Process): def startQtEventLoop(name, port, authkey, ppid, debug=False): if debug: import os - print('[%d] connecting to server at port localhost:%d, authkey=%s..' % (os.getpid(), port, repr(authkey))) + cprint.cout(debug, '[%d] connecting to server at port localhost:%d, authkey=%s..\n' % (os.getpid(), port, repr(authkey)), -1) conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey) if debug: - print('[%d] connected; starting remote proxy.' % os.getpid()) + cprint.cout(debug, '[%d] connected; starting remote proxy.\n' % os.getpid(), -1) from ..Qt import QtGui, QtCore #from PyQt4 import QtGui, QtCore app = QtGui.QApplication.instance() @@ -437,11 +448,13 @@ class FileForwarder(threading.Thread): which ensures that the correct behavior is achieved even if sys.stdout/stderr are replaced at runtime. """ - def __init__(self, input, output): + def __init__(self, input, output, color): threading.Thread.__init__(self) self.input = input self.output = output self.lock = threading.Lock() + self.daemon = True + self.color = color self.start() def run(self): @@ -449,12 +462,12 @@ class FileForwarder(threading.Thread): while True: line = self.input.readline() with self.lock: - sys.stdout.write(line) + cprint.cout(self.color, line, -1) elif self.output == 'stderr': while True: line = self.input.readline() with self.lock: - sys.stderr.write(line) + cprint.cerr(self.color, line, -1) else: while True: line = self.input.readline() diff --git a/pyqtgraph/multiprocess/remoteproxy.py b/pyqtgraph/multiprocess/remoteproxy.py index 70ce90a6..8287d0e8 100644 --- a/pyqtgraph/multiprocess/remoteproxy.py +++ b/pyqtgraph/multiprocess/remoteproxy.py @@ -7,6 +7,9 @@ except ImportError: import builtins import pickle +# color printing for debugging +from ..util import cprint + class ClosedError(Exception): """Raised when an event handler receives a request to close the connection or discovers that the connection has been closed.""" @@ -80,7 +83,7 @@ class RemoteEventHandler(object): def debugMsg(self, msg): if not self.debug: return - print("[%d] %s" % (os.getpid(), str(msg))) + cprint.cout(self.debug, "%d [%d] %s\n" % (self.debug, os.getpid(), str(msg)), -1) def getProxyOption(self, opt): return self.proxyOptions[opt] diff --git a/pyqtgraph/util/Mutex.py b/pyqtgraph/util/Mutex.py new file mode 100644 index 00000000..8335a812 --- /dev/null +++ b/pyqtgraph/util/Mutex.py @@ -0,0 +1,277 @@ +# -*- coding: utf-8 -*- +""" +Mutex.py - Stand-in extension of Qt's QMutex class +Copyright 2010 Luke Campagnola +Distributed under MIT/X11 license. See license.txt for more infomation. +""" + +from PyQt4 import QtCore +import traceback + +class Mutex(QtCore.QMutex): + """Extends QMutex to provide warning messages when a mutex stays locked for a long time. + Mostly just useful for debugging purposes. Should only be used with MutexLocker, not + QMutexLocker. + """ + + def __init__(self, *args, **kargs): + if kargs.get('recursive', False): + args = (QtCore.QMutex.Recursive,) + QtCore.QMutex.__init__(self, *args) + self.l = QtCore.QMutex() ## for serializing access to self.tb + self.tb = [] + self.debug = False ## True to enable debugging functions + + def tryLock(self, timeout=None, id=None): + if timeout is None: + locked = QtCore.QMutex.tryLock(self) + else: + locked = QtCore.QMutex.tryLock(self, timeout) + + if self.debug and locked: + self.l.lock() + try: + if id is None: + self.tb.append(''.join(traceback.format_stack()[:-1])) + else: + self.tb.append(" " + str(id)) + #print 'trylock', self, len(self.tb) + finally: + self.l.unlock() + return locked + + def lock(self, id=None): + c = 0 + waitTime = 5000 # in ms + while True: + if self.tryLock(waitTime, id): + break + c += 1 + if self.debug: + self.l.lock() + try: + print "Waiting for mutex lock (%0.1f sec). Traceback follows:" % (c*waitTime/1000.) + traceback.print_stack() + if len(self.tb) > 0: + print "Mutex is currently locked from:\n", self.tb[-1] + else: + print "Mutex is currently locked from [???]" + finally: + self.l.unlock() + #print 'lock', self, len(self.tb) + + def unlock(self): + QtCore.QMutex.unlock(self) + if self.debug: + self.l.lock() + try: + #print 'unlock', self, len(self.tb) + if len(self.tb) > 0: + self.tb.pop() + else: + raise Exception("Attempt to unlock mutex before it has been locked") + finally: + self.l.unlock() + + def depth(self): + self.l.lock() + n = len(self.tb) + self.l.unlock() + return n + + def traceback(self): + self.l.lock() + try: + ret = self.tb[:] + finally: + self.l.unlock() + return ret + + def __exit__(self, *args): + self.unlock() + + def __enter__(self): + self.lock() + return self + + +class MutexLocker: + def __init__(self, lock): + #print self, "lock on init",lock, lock.depth() + self.lock = lock + self.lock.lock() + self.unlockOnDel = True + + def unlock(self): + #print self, "unlock by req",self.lock, self.lock.depth() + self.lock.unlock() + self.unlockOnDel = False + + + def relock(self): + #print self, "relock by req",self.lock, self.lock.depth() + self.lock.lock() + self.unlockOnDel = True + + def __del__(self): + if self.unlockOnDel: + #print self, "Unlock by delete:", self.lock, self.lock.depth() + self.lock.unlock() + #else: + #print self, "Skip unlock by delete", self.lock, self.lock.depth() + + def __exit__(self, *args): + if self.unlockOnDel: + self.unlock() + + def __enter__(self): + return self + + def mutex(self): + return self.lock + +#import functools +#def methodWrapper(fn, self, *args, **kargs): + #print repr(fn), repr(self), args, kargs + #obj = self.__wrapped_object__() + #return getattr(obj, fn)(*args, **kargs) + +##def WrapperClass(clsName, parents, attrs): + ##for parent in parents: + ##for name in dir(parent): + ##attr = getattr(parent, name) + ##if callable(attr) and name not in attrs: + ##attrs[name] = functools.partial(funcWrapper, name) + ##return type(clsName, parents, attrs) + +#def WrapperClass(name, bases, attrs): + #for n in ['__getattr__', '__setattr__', '__getitem__', '__setitem__']: + #if n not in attrs: + #attrs[n] = functools.partial(methodWrapper, n) + #return type(name, bases, attrs) + +#class WrapperClass(type): + #def __new__(cls, name, bases, attrs): + #fakes = [] + #for n in ['__getitem__', '__setitem__']: + #if n not in attrs: + #attrs[n] = lambda self, *args: getattr(self, n)(*args) + #fakes.append(n) + #print fakes + #typ = type(name, bases, attrs) + #typ.__faked_methods__ = fakes + #return typ + + #def __init__(self, name, bases, attrs): + #print self.__faked_methods__ + #for n in self.__faked_methods__: + #self.n = None + + + +#class ThreadsafeWrapper(object): + #def __init__(self, obj): + #self.__TSW_object__ = obj + + #def __wrapped_object__(self): + #return self.__TSW_object__ + + +class ThreadsafeWrapper(object): + """Wrapper that makes access to any object thread-safe (within reasonable limits). + Mostly tested for wrapping lists, dicts, etc. + NOTE: Do not instantiate directly; use threadsafe(obj) instead. + - all method calls and attribute/item accesses are protected by mutex + - optionally, attribute/item accesses may return protected objects + - can be manually locked for extended operations + """ + def __init__(self, obj, recursive=False, reentrant=True): + """ + If recursive is True, then sub-objects accessed from obj are wrapped threadsafe as well. + If reentrant is True, then the object can be locked multiple times from the same thread.""" + + self.__TSOwrapped_object__ = obj + + if reentrant: + self.__TSOwrap_lock__ = Mutex(QtCore.QMutex.Recursive) + else: + self.__TSOwrap_lock__ = Mutex() + self.__TSOrecursive__ = recursive + self.__TSOreentrant__ = reentrant + self.__TSOwrapped_objs__ = {} + + def lock(self, id=None): + self.__TSOwrap_lock__.lock(id=id) + + def tryLock(self, timeout=None, id=None): + self.__TSOwrap_lock__.tryLock(timeout=timeout, id=id) + + def unlock(self): + self.__TSOwrap_lock__.unlock() + + def unwrap(self): + return self.__TSOwrapped_object__ + + def __safe_call__(self, fn, *args, **kargs): + obj = self.__wrapped_object__() + ret = getattr(obj, fn)(*args, **kargs) + return self.__wrap_object__(ret) + + def __getattr__(self, attr): + #try: + #return object.__getattribute__(self, attr) + #except AttributeError: + with self.__TSOwrap_lock__: + val = getattr(self.__wrapped_object__(), attr) + #if callable(val): + #return self.__wrap_object__(val) + return self.__wrap_object__(val) + + def __setattr__(self, attr, val): + if attr[:5] == '__TSO': + #return object.__setattr__(self, attr, val) + self.__dict__[attr] = val + return + with self.__TSOwrap_lock__: + return setattr(self.__wrapped_object__(), attr, val) + + def __wrap_object__(self, obj): + if not self.__TSOrecursive__: + return obj + if obj.__class__ in [int, float, str, unicode, tuple]: + return obj + if id(obj) not in self.__TSOwrapped_objs__: + self.__TSOwrapped_objs__[id(obj)] = threadsafe(obj, recursive=self.__TSOrecursive__, reentrant=self.__TSOreentrant__) + return self.__TSOwrapped_objs__[id(obj)] + + def __wrapped_object__(self): + #if isinstance(self.__TSOwrapped_object__, weakref.ref): + #return self.__TSOwrapped_object__() + #else: + return self.__TSOwrapped_object__ + +def mkMethodWrapper(name): + return lambda self, *args, **kargs: self.__safe_call__(name, *args, **kargs) + +def threadsafe(obj, *args, **kargs): + """Return a thread-safe wrapper around obj. (see ThreadsafeWrapper) + args and kargs are passed directly to ThreadsafeWrapper.__init__() + This factory function is necessary for wrapping special methods (like __getitem__)""" + if type(obj) in [int, float, str, unicode, tuple, type(None), bool]: + return obj + clsName = 'Threadsafe_' + obj.__class__.__name__ + attrs = {} + ignore = set(['__new__', '__init__', '__class__', '__hash__', '__getattribute__', '__getattr__', '__setattr__']) + for n in dir(obj): + if not n.startswith('__') or n in ignore: + continue + v = getattr(obj, n) + if callable(v): + attrs[n] = mkMethodWrapper(n) + typ = type(clsName, (ThreadsafeWrapper,), attrs) + return typ(obj, *args, **kargs) + + +if __name__ == '__main__': + d = {'x': 3, 'y': [1,2,3,4], 'z': {'a': 3}, 'w': (1,2,3,4)} + t = threadsafe(d, recursive=True, reentrant=False) \ No newline at end of file diff --git a/pyqtgraph/util/colorama/LICENSE.txt b/pyqtgraph/util/colorama/LICENSE.txt new file mode 100644 index 00000000..5f567799 --- /dev/null +++ b/pyqtgraph/util/colorama/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright (c) 2010 Jonathan Hartley +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holders, nor those of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/pyqtgraph/util/colorama/README.txt b/pyqtgraph/util/colorama/README.txt new file mode 100644 index 00000000..8910ba5b --- /dev/null +++ b/pyqtgraph/util/colorama/README.txt @@ -0,0 +1,304 @@ +Download and docs: + http://pypi.python.org/pypi/colorama +Development: + http://code.google.com/p/colorama +Discussion group: + https://groups.google.com/forum/#!forum/python-colorama + +Description +=========== + +Makes ANSI escape character sequences for producing colored terminal text and +cursor positioning work under MS Windows. + +ANSI escape character sequences have long been used to produce colored terminal +text and cursor positioning on Unix and Macs. Colorama makes this work on +Windows, too, by wrapping stdout, stripping ANSI sequences it finds (which +otherwise show up as gobbledygook in your output), and converting them into the +appropriate win32 calls to modify the state of the terminal. On other platforms, +Colorama does nothing. + +Colorama also provides some shortcuts to help generate ANSI sequences +but works fine in conjunction with any other ANSI sequence generation library, +such as Termcolor (http://pypi.python.org/pypi/termcolor.) + +This has the upshot of providing a simple cross-platform API for printing +colored terminal text from Python, and has the happy side-effect that existing +applications or libraries which use ANSI sequences to produce colored output on +Linux or Macs can now also work on Windows, simply by calling +``colorama.init()``. + +An alternative approach is to install 'ansi.sys' on Windows machines, which +provides the same behaviour for all applications running in terminals. Colorama +is intended for situations where that isn't easy (e.g. maybe your app doesn't +have an installer.) + +Demo scripts in the source code repository prints some colored text using +ANSI sequences. Compare their output under Gnome-terminal's built in ANSI +handling, versus on Windows Command-Prompt using Colorama: + +.. image:: http://colorama.googlecode.com/hg/screenshots/ubuntu-demo.png + :width: 661 + :height: 357 + :alt: ANSI sequences on Ubuntu under gnome-terminal. + +.. image:: http://colorama.googlecode.com/hg/screenshots/windows-demo.png + :width: 668 + :height: 325 + :alt: Same ANSI sequences on Windows, using Colorama. + +These screengrabs show that Colorama on Windows does not support ANSI 'dim +text': it looks the same as 'normal text'. + + +License +======= + +Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. + + +Dependencies +============ + +None, other than Python. Tested on Python 2.5.5, 2.6.5, 2.7, 3.1.2, and 3.2 + +Usage +===== + +Initialisation +-------------- + +Applications should initialise Colorama using:: + + from colorama import init + init() + +If you are on Windows, the call to ``init()`` will start filtering ANSI escape +sequences out of any text sent to stdout or stderr, and will replace them with +equivalent Win32 calls. + +Calling ``init()`` has no effect on other platforms (unless you request other +optional functionality, see keyword args below.) The intention is that +applications can call ``init()`` unconditionally on all platforms, after which +ANSI output should just work. + +To stop using colorama before your program exits, simply call ``deinit()``. +This will restore stdout and stderr to their original values, so that Colorama +is disabled. To start using Colorama again, call ``reinit()``, which wraps +stdout and stderr again, but is cheaper to call than doing ``init()`` all over +again. + + +Colored Output +-------------- + +Cross-platform printing of colored text can then be done using Colorama's +constant shorthand for ANSI escape sequences:: + + from colorama import Fore, Back, Style + print(Fore.RED + 'some red text') + print(Back.GREEN + 'and with a green background') + print(Style.DIM + 'and in dim text') + print(Fore.RESET + Back.RESET + Style.RESET_ALL) + print('back to normal now') + +or simply by manually printing ANSI sequences from your own code:: + + print('/033[31m' + 'some red text') + print('/033[30m' # and reset to default color) + +or Colorama can be used happily in conjunction with existing ANSI libraries +such as Termcolor:: + + from colorama import init + from termcolor import colored + + # use Colorama to make Termcolor work on Windows too + init() + + # then use Termcolor for all colored text output + print(colored('Hello, World!', 'green', 'on_red')) + +Available formatting constants are:: + + Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. + Style: DIM, NORMAL, BRIGHT, RESET_ALL + +Style.RESET_ALL resets foreground, background and brightness. Colorama will +perform this reset automatically on program exit. + + +Cursor Positioning +------------------ + +ANSI codes to reposition the cursor are supported. See demos/demo06.py for +an example of how to generate them. + + +Init Keyword Args +----------------- + +``init()`` accepts some kwargs to override default behaviour. + +init(autoreset=False): + If you find yourself repeatedly sending reset sequences to turn off color + changes at the end of every print, then ``init(autoreset=True)`` will + automate that:: + + from colorama import init + init(autoreset=True) + print(Fore.RED + 'some red text') + print('automatically back to default color again') + +init(strip=None): + Pass ``True`` or ``False`` to override whether ansi codes should be + stripped from the output. The default behaviour is to strip if on Windows. + +init(convert=None): + Pass ``True`` or ``False`` to override whether to convert ansi codes in the + output into win32 calls. The default behaviour is to convert if on Windows + and output is to a tty (terminal). + +init(wrap=True): + On Windows, colorama works by replacing ``sys.stdout`` and ``sys.stderr`` + with proxy objects, which override the .write() method to do their work. If + this wrapping causes you problems, then this can be disabled by passing + ``init(wrap=False)``. The default behaviour is to wrap if autoreset or + strip or convert are True. + + When wrapping is disabled, colored printing on non-Windows platforms will + continue to work as normal. To do cross-platform colored output, you can + use Colorama's ``AnsiToWin32`` proxy directly:: + + import sys + from colorama import init, AnsiToWin32 + init(wrap=False) + stream = AnsiToWin32(sys.stderr).stream + + # Python 2 + print >>stream, Fore.BLUE + 'blue text on stderr' + + # Python 3 + print(Fore.BLUE + 'blue text on stderr', file=stream) + + +Status & Known Problems +======================= + +I've personally only tested it on WinXP (CMD, Console2), Ubuntu +(gnome-terminal, xterm), and OSX. + +Some presumably valid ANSI sequences aren't recognised (see details below) +but to my knowledge nobody has yet complained about this. Puzzling. + +See outstanding issues and wishlist at: +http://code.google.com/p/colorama/issues/list + +If anything doesn't work for you, or doesn't do what you expected or hoped for, +I'd love to hear about it on that issues list, would be delighted by patches, +and would be happy to grant commit access to anyone who submits a working patch +or two. + + +Recognised ANSI Sequences +========================= + +ANSI sequences generally take the form: + + ESC [ ; ... + +Where is an integer, and is a single letter. Zero or more +params are passed to a . If no params are passed, it is generally +synonymous with passing a single zero. No spaces exist in the sequence, they +have just been inserted here to make it easy to read. + +The only ANSI sequences that colorama converts into win32 calls are:: + + ESC [ 0 m # reset all (colors and brightness) + ESC [ 1 m # bright + ESC [ 2 m # dim (looks same as normal brightness) + ESC [ 22 m # normal brightness + + # FOREGROUND: + ESC [ 30 m # black + ESC [ 31 m # red + ESC [ 32 m # green + ESC [ 33 m # yellow + ESC [ 34 m # blue + ESC [ 35 m # magenta + ESC [ 36 m # cyan + ESC [ 37 m # white + ESC [ 39 m # reset + + # BACKGROUND + ESC [ 40 m # black + ESC [ 41 m # red + ESC [ 42 m # green + ESC [ 43 m # yellow + ESC [ 44 m # blue + ESC [ 45 m # magenta + ESC [ 46 m # cyan + ESC [ 47 m # white + ESC [ 49 m # reset + + # cursor positioning + ESC [ y;x H # position cursor at x across, y down + + # clear the screen + ESC [ mode J # clear the screen. Only mode 2 (clear entire screen) + # is supported. It should be easy to add other modes, + # let me know if that would be useful. + +Multiple numeric params to the 'm' command can be combined into a single +sequence, eg:: + + ESC [ 36 ; 45 ; 1 m # bright cyan text on magenta background + +All other ANSI sequences of the form ``ESC [ ; ... `` +are silently stripped from the output on Windows. + +Any other form of ANSI sequence, such as single-character codes or alternative +initial characters, are not recognised nor stripped. It would be cool to add +them though. Let me know if it would be useful for you, via the issues on +google code. + + +Development +=========== + +Help and fixes welcome! Ask Jonathan for commit rights, you'll get them. + +Running tests requires: + +- Michael Foord's 'mock' module to be installed. +- Tests are written using the 2010 era updates to 'unittest', and require to + be run either using Python2.7 or greater, or else to have Michael Foord's + 'unittest2' module installed. + +unittest2 test discovery doesn't work for colorama, so I use 'nose':: + + nosetests -s + +The -s is required because 'nosetests' otherwise applies a proxy of its own to +stdout, which confuses the unit tests. + + +Contact +======= + +Created by Jonathan Hartley, tartley@tartley.com + + +Thanks +====== +| Ben Hoyt, for a magnificent fix under 64-bit Windows. +| Jesse@EmptySquare for submitting a fix for examples in the README. +| User 'jamessp', an observant documentation fix for cursor positioning. +| User 'vaal1239', Dave Mckee & Lackner Kristof for a tiny but much-needed Win7 fix. +| Julien Stuyck, for wisely suggesting Python3 compatible updates to README. +| Daniel Griffith for multiple fabulous patches. +| Oscar Lesta for valuable fix to stop ANSI chars being sent to non-tty output. +| Roger Binns, for many suggestions, valuable feedback, & bug reports. +| Tim Golden for thought and much appreciated feedback on the initial idea. + diff --git a/pyqtgraph/util/colorama/__init__.py b/pyqtgraph/util/colorama/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pyqtgraph/util/colorama/win32.py b/pyqtgraph/util/colorama/win32.py new file mode 100644 index 00000000..f4024f95 --- /dev/null +++ b/pyqtgraph/util/colorama/win32.py @@ -0,0 +1,134 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. + +# from winbase.h +STDOUT = -11 +STDERR = -12 + +try: + from ctypes import windll + from ctypes import wintypes +except ImportError: + windll = None + SetConsoleTextAttribute = lambda *_: None +else: + from ctypes import ( + byref, Structure, c_char, c_short, c_uint32, c_ushort, POINTER + ) + + class CONSOLE_SCREEN_BUFFER_INFO(Structure): + """struct in wincon.h.""" + _fields_ = [ + ("dwSize", wintypes._COORD), + ("dwCursorPosition", wintypes._COORD), + ("wAttributes", wintypes.WORD), + ("srWindow", wintypes.SMALL_RECT), + ("dwMaximumWindowSize", wintypes._COORD), + ] + def __str__(self): + return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % ( + self.dwSize.Y, self.dwSize.X + , self.dwCursorPosition.Y, self.dwCursorPosition.X + , self.wAttributes + , self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right + , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X + ) + + _GetStdHandle = windll.kernel32.GetStdHandle + _GetStdHandle.argtypes = [ + wintypes.DWORD, + ] + _GetStdHandle.restype = wintypes.HANDLE + + _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo + _GetConsoleScreenBufferInfo.argtypes = [ + wintypes.HANDLE, + POINTER(CONSOLE_SCREEN_BUFFER_INFO), + ] + _GetConsoleScreenBufferInfo.restype = wintypes.BOOL + + _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute + _SetConsoleTextAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + ] + _SetConsoleTextAttribute.restype = wintypes.BOOL + + _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition + _SetConsoleCursorPosition.argtypes = [ + wintypes.HANDLE, + wintypes._COORD, + ] + _SetConsoleCursorPosition.restype = wintypes.BOOL + + _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA + _FillConsoleOutputCharacterA.argtypes = [ + wintypes.HANDLE, + c_char, + wintypes.DWORD, + wintypes._COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputCharacterA.restype = wintypes.BOOL + + _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute + _FillConsoleOutputAttribute.argtypes = [ + wintypes.HANDLE, + wintypes.WORD, + wintypes.DWORD, + wintypes._COORD, + POINTER(wintypes.DWORD), + ] + _FillConsoleOutputAttribute.restype = wintypes.BOOL + + handles = { + STDOUT: _GetStdHandle(STDOUT), + STDERR: _GetStdHandle(STDERR), + } + + def GetConsoleScreenBufferInfo(stream_id=STDOUT): + handle = handles[stream_id] + csbi = CONSOLE_SCREEN_BUFFER_INFO() + success = _GetConsoleScreenBufferInfo( + handle, byref(csbi)) + return csbi + + def SetConsoleTextAttribute(stream_id, attrs): + handle = handles[stream_id] + return _SetConsoleTextAttribute(handle, attrs) + + def SetConsoleCursorPosition(stream_id, position): + position = wintypes._COORD(*position) + # If the position is out of range, do nothing. + if position.Y <= 0 or position.X <= 0: + return + # Adjust for Windows' SetConsoleCursorPosition: + # 1. being 0-based, while ANSI is 1-based. + # 2. expecting (x,y), while ANSI uses (y,x). + adjusted_position = wintypes._COORD(position.Y - 1, position.X - 1) + # Adjust for viewport's scroll position + sr = GetConsoleScreenBufferInfo(STDOUT).srWindow + adjusted_position.Y += sr.Top + adjusted_position.X += sr.Left + # Resume normal processing + handle = handles[stream_id] + return _SetConsoleCursorPosition(handle, adjusted_position) + + def FillConsoleOutputCharacter(stream_id, char, length, start): + handle = handles[stream_id] + char = c_char(char) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + success = _FillConsoleOutputCharacterA( + handle, char, length, start, byref(num_written)) + return num_written.value + + def FillConsoleOutputAttribute(stream_id, attr, length, start): + ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' + handle = handles[stream_id] + attribute = wintypes.WORD(attr) + length = wintypes.DWORD(length) + num_written = wintypes.DWORD(0) + # Note that this is hard-coded for ANSI (vs wide) bytes. + return _FillConsoleOutputAttribute( + handle, attribute, length, start, byref(num_written)) diff --git a/pyqtgraph/util/colorama/winterm.py b/pyqtgraph/util/colorama/winterm.py new file mode 100644 index 00000000..27088115 --- /dev/null +++ b/pyqtgraph/util/colorama/winterm.py @@ -0,0 +1,120 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from . import win32 + + +# from wincon.h +class WinColor(object): + BLACK = 0 + BLUE = 1 + GREEN = 2 + CYAN = 3 + RED = 4 + MAGENTA = 5 + YELLOW = 6 + GREY = 7 + +# from wincon.h +class WinStyle(object): + NORMAL = 0x00 # dim text, dim background + BRIGHT = 0x08 # bright text, dim background + + +class WinTerm(object): + + def __init__(self): + self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes + self.set_attrs(self._default) + self._default_fore = self._fore + self._default_back = self._back + self._default_style = self._style + + def get_attrs(self): + return self._fore + self._back * 16 + self._style + + def set_attrs(self, value): + self._fore = value & 7 + self._back = (value >> 4) & 7 + self._style = value & WinStyle.BRIGHT + + def reset_all(self, on_stderr=None): + self.set_attrs(self._default) + self.set_console(attrs=self._default) + + def fore(self, fore=None, on_stderr=False): + if fore is None: + fore = self._default_fore + self._fore = fore + self.set_console(on_stderr=on_stderr) + + def back(self, back=None, on_stderr=False): + if back is None: + back = self._default_back + self._back = back + self.set_console(on_stderr=on_stderr) + + def style(self, style=None, on_stderr=False): + if style is None: + style = self._default_style + self._style = style + self.set_console(on_stderr=on_stderr) + + def set_console(self, attrs=None, on_stderr=False): + if attrs is None: + attrs = self.get_attrs() + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleTextAttribute(handle, attrs) + + def get_position(self, handle): + position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition + # Because Windows coordinates are 0-based, + # and win32.SetConsoleCursorPosition expects 1-based. + position.X += 1 + position.Y += 1 + return position + + def set_cursor_position(self, position=None, on_stderr=False): + if position is None: + #I'm not currently tracking the position, so there is no default. + #position = self.get_position() + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + win32.SetConsoleCursorPosition(handle, position) + + def cursor_up(self, num_rows=0, on_stderr=False): + if num_rows == 0: + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + position = self.get_position(handle) + adjusted_position = (position.Y - num_rows, position.X) + self.set_cursor_position(adjusted_position, on_stderr) + + def erase_data(self, mode=0, on_stderr=False): + # 0 (or None) should clear from the cursor to the end of the screen. + # 1 should clear from the cursor to the beginning of the screen. + # 2 should clear the entire screen. (And maybe move cursor to (1,1)?) + # + # At the moment, I only support mode 2. From looking at the API, it + # should be possible to calculate a different number of bytes to clear, + # and to do so relative to the cursor position. + if mode[0] not in (2,): + return + handle = win32.STDOUT + if on_stderr: + handle = win32.STDERR + # here's where we'll home the cursor + coord_screen = win32.COORD(0,0) + csbi = win32.GetConsoleScreenBufferInfo(handle) + # get the number of character cells in the current buffer + dw_con_size = csbi.dwSize.X * csbi.dwSize.Y + # fill the entire screen with blanks + win32.FillConsoleOutputCharacter(handle, ' ', dw_con_size, coord_screen) + # now set the buffer's attributes accordingly + win32.FillConsoleOutputAttribute(handle, self.get_attrs(), dw_con_size, coord_screen ); + # put the cursor at (0, 0) + win32.SetConsoleCursorPosition(handle, (coord_screen.X, coord_screen.Y)) diff --git a/pyqtgraph/util/cprint.py b/pyqtgraph/util/cprint.py new file mode 100644 index 00000000..e88bfd1a --- /dev/null +++ b/pyqtgraph/util/cprint.py @@ -0,0 +1,101 @@ +""" +Cross-platform color text printing + +Based on colorama (see pyqtgraph/util/colorama/README.txt) +""" +import sys, re + +from .colorama.winterm import WinTerm, WinColor, WinStyle +from .colorama.win32 import windll + +_WIN = sys.platform.startswith('win') +if windll is not None: + winterm = WinTerm() +else: + _WIN = False + +def winset(reset=False, fore=None, back=None, style=None, stderr=False): + if reset: + winterm.reset_all() + if fore is not None: + winterm.fore(fore, stderr) + if back is not None: + winterm.back(back, stderr) + if style is not None: + winterm.style(style, stderr) + +ANSI = {} +WIN = {} +for i,color in enumerate(['BLACK', 'RED', 'GREEN', 'YELLOW', 'BLUE', 'MAGENTA', 'CYAN', 'WHITE']): + globals()[color] = i + globals()['BR_' + color] = i + 8 + globals()['BACK_' + color] = i + 40 + ANSI[i] = "\033[%dm" % (30+i) + ANSI[i+8] = "\033[2;%dm" % (30+i) + ANSI[i+40] = "\033[%dm" % (40+i) + color = 'GREY' if color == 'WHITE' else color + WIN[i] = {'fore': getattr(WinColor, color), 'style': WinStyle.NORMAL} + WIN[i+8] = {'fore': getattr(WinColor, color), 'style': WinStyle.BRIGHT} + WIN[i+40] = {'back': getattr(WinColor, color)} + +RESET = -1 +ANSI[RESET] = "\033[0m" +WIN[RESET] = {'reset': True} + + +def cprint(stream, *args, **kwds): + """ + Print with color. Examples:: + + # colors are BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE + cprint('stdout', RED, 'This is in red. ', RESET, 'and this is normal\n') + + # Adding BR_ before the color manes it bright + cprint('stdout', BR_GREEN, 'This is bright green.\n', RESET) + + # Adding BACK_ changes background color + cprint('stderr', BACK_BLUE, WHITE, 'This is white-on-blue.', -1) + + # Integers 0-7 for normal, 8-15 for bright, and 40-47 for background. + # -1 to reset. + cprint('stderr', 1, 'This is in red.', -1) + + """ + if isinstance(stream, basestring): + stream = kwds.get('stream', 'stdout') + err = stream == 'stderr' + stream = getattr(sys, stream) + else: + err = kwds.get('stderr', False) + + if hasattr(stream, 'isatty') and stream.isatty(): + if _WIN: + # convert to win32 calls + for arg in args: + if isinstance(arg, basestring): + stream.write(arg) + else: + kwds = WIN[arg] + winset(stderr=err, **kwds) + else: + # convert to ANSI + for arg in args: + if isinstance(arg, basestring): + stream.write(arg) + else: + stream.write(ANSI[arg]) + else: + # ignore colors + for arg in args: + if isinstance(arg, basestring): + stream.write(arg) + +def cout(*args): + """Shorthand for cprint('stdout', ...)""" + cprint('stdout', *args) + +def cerr(*args): + """Shorthand for cprint('stderr', ...)""" + cprint('stderr', *args) + + diff --git a/pyqtgraph/widgets/RemoteGraphicsView.py b/pyqtgraph/widgets/RemoteGraphicsView.py index 54712f43..cb9a7052 100644 --- a/pyqtgraph/widgets/RemoteGraphicsView.py +++ b/pyqtgraph/widgets/RemoteGraphicsView.py @@ -3,6 +3,7 @@ if not USE_PYSIDE: import sip from .. import multiprocess as mp from .GraphicsView import GraphicsView +from .. import CONFIG_OPTIONS import numpy as np import mmap, tempfile, ctypes, atexit, sys, random @@ -35,7 +36,7 @@ class RemoteGraphicsView(QtGui.QWidget): self._proc = mp.QtProcess(**kwds) self.pg = self._proc._import('pyqtgraph') - self.pg.setConfigOptions(**self.pg.CONFIG_OPTIONS) + self.pg.setConfigOptions(**CONFIG_OPTIONS) rpgRemote = self._proc._import('pyqtgraph.widgets.RemoteGraphicsView') self._view = rpgRemote.Renderer(*args, **remoteKwds) self._view._setProxyOptions(deferGetattr=True)