Import image testing code from vispy
This commit is contained in:
parent
fd76443458
commit
89cb6e4108
434
pyqtgraph/tests/image_testing.py
Normal file
434
pyqtgraph/tests/image_testing.py
Normal file
|
@ -0,0 +1,434 @@
|
||||||
|
# Image-based testing borrowed from vispy
|
||||||
|
|
||||||
|
"""
|
||||||
|
Procedure for unit-testing with images:
|
||||||
|
|
||||||
|
1. Run unit tests at least once; this initializes a git clone of
|
||||||
|
pyqtgraph/test-data in ~/.pyqtgraph.
|
||||||
|
|
||||||
|
2. Run individual test scripts with the PYQTGRAPH_AUDIT environment variable set:
|
||||||
|
|
||||||
|
$ PYQTGRAPH_AUDIT=1 python pyqtgraph/graphicsItems/tests/test_PlotItem.py
|
||||||
|
|
||||||
|
Any failing tests will
|
||||||
|
display the test results, standard image, and the differences between the
|
||||||
|
two. If the test result is bad, then press (f)ail. If the test result is
|
||||||
|
good, then press (p)ass and the new image will be saved to the test-data
|
||||||
|
directory.
|
||||||
|
|
||||||
|
3. After adding or changing test images, create a new commit:
|
||||||
|
|
||||||
|
$ cd ~/.pyqtgraph/test-data
|
||||||
|
$ git add ...
|
||||||
|
$ git commit -a
|
||||||
|
|
||||||
|
4. Look up the most recent tag name from the `test_data_tag` variable in
|
||||||
|
get_test_data_repo() below. Increment the tag name by 1 in the function
|
||||||
|
and create a new tag in the test-data repository:
|
||||||
|
|
||||||
|
$ git tag test-data-NNN
|
||||||
|
$ git push --tags origin master
|
||||||
|
|
||||||
|
This tag is used to ensure that each pyqtgraph commit is linked to a specific
|
||||||
|
commit in the test-data repository. This makes it possible to push new
|
||||||
|
commits to the test-data repository without interfering with existing
|
||||||
|
tests, and also allows unit tests to continue working on older pyqtgraph
|
||||||
|
versions.
|
||||||
|
|
||||||
|
Finally, update the tag name in ``get_test_data_repo`` to the new name.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import inspect
|
||||||
|
import base64
|
||||||
|
from subprocess import check_call, CalledProcessError
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from ..ext.six.moves import http_client as httplib
|
||||||
|
from ..ext.six.moves import urllib_parse as urllib
|
||||||
|
from .. import scene, config
|
||||||
|
from ..util import run_subprocess
|
||||||
|
|
||||||
|
|
||||||
|
tester = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_tester():
|
||||||
|
global tester
|
||||||
|
if tester is None:
|
||||||
|
tester = ImageTester()
|
||||||
|
return tester
|
||||||
|
|
||||||
|
|
||||||
|
def assert_image_approved(image, standard_file, message=None, **kwargs):
|
||||||
|
"""Check that an image test result matches a pre-approved standard.
|
||||||
|
|
||||||
|
If the result does not match, then the user can optionally invoke a GUI
|
||||||
|
to compare the images and decide whether to fail the test or save the new
|
||||||
|
image as the standard.
|
||||||
|
|
||||||
|
This function will automatically clone the test-data repository into
|
||||||
|
~/.pyqtgraph/test-data. However, it is up to the user to ensure this repository
|
||||||
|
is kept up to date and to commit/push new images after they are saved.
|
||||||
|
|
||||||
|
Run the test with the environment variable PYQTGRAPH_AUDIT=1 to bring up
|
||||||
|
the auditing GUI.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
image : (h, w, 4) ndarray
|
||||||
|
standard_file : str
|
||||||
|
The name of the approved test image to check against. This file name
|
||||||
|
is relative to the root of the pyqtgraph test-data repository and will
|
||||||
|
be automatically fetched.
|
||||||
|
message : str
|
||||||
|
A string description of the image. It is recommended to describe
|
||||||
|
specific features that an auditor should look for when deciding whether
|
||||||
|
to fail a test.
|
||||||
|
|
||||||
|
Extra keyword arguments are used to set the thresholds for automatic image
|
||||||
|
comparison (see ``assert_image_match()``).
|
||||||
|
"""
|
||||||
|
|
||||||
|
if message is None:
|
||||||
|
code = inspect.currentframe().f_back.f_code
|
||||||
|
message = "%s::%s" % (code.co_filename, code.co_name)
|
||||||
|
|
||||||
|
# Make sure we have a test data repo available, possibly invoking git
|
||||||
|
data_path = get_test_data_repo()
|
||||||
|
|
||||||
|
# Read the standard image if it exists
|
||||||
|
std_file = os.path.join(data_path, standard_file)
|
||||||
|
if not os.path.isfile(std_file):
|
||||||
|
std_image = None
|
||||||
|
else:
|
||||||
|
std_image = read_png(std_file)
|
||||||
|
|
||||||
|
# If the test image does not match, then we go to audit if requested.
|
||||||
|
try:
|
||||||
|
if image.shape != std_image.shape:
|
||||||
|
# Allow im1 to be an integer multiple larger than im2 to account
|
||||||
|
# for high-resolution displays
|
||||||
|
ims1 = np.array(image.shape).astype(float)
|
||||||
|
ims2 = np.array(std_image.shape).astype(float)
|
||||||
|
sr = ims1 / ims2
|
||||||
|
if (sr[0] != sr[1] or not np.allclose(sr, np.round(sr)) or
|
||||||
|
sr[0] < 1):
|
||||||
|
raise TypeError("Test result shape %s is not an integer factor"
|
||||||
|
" larger than standard image shape %s." %
|
||||||
|
(ims1, ims2))
|
||||||
|
sr = np.round(sr).astype(int)
|
||||||
|
image = downsample(image, sr[0], axis=(0, 1)).astype(image.dtype)
|
||||||
|
|
||||||
|
assert_image_match(image, std_image, **kwargs)
|
||||||
|
except Exception:
|
||||||
|
if standard_file in git_status(data_path):
|
||||||
|
print("\n\nWARNING: unit test failed against modified standard "
|
||||||
|
"image %s.\nTo revert this file, run `cd %s; git checkout "
|
||||||
|
"%s`\n" % (std_file, data_path, standard_file))
|
||||||
|
if os.getenv('PYQTGRAPH_AUDIT') == '1':
|
||||||
|
sys.excepthook(*sys.exc_info())
|
||||||
|
_get_tester().test(image, std_image, message)
|
||||||
|
std_path = os.path.dirname(std_file)
|
||||||
|
print('Saving new standard image to "%s"' % std_file)
|
||||||
|
if not os.path.isdir(std_path):
|
||||||
|
os.makedirs(std_path)
|
||||||
|
write_png(std_file, image)
|
||||||
|
else:
|
||||||
|
if std_image is None:
|
||||||
|
raise Exception("Test standard %s does not exist." % std_file)
|
||||||
|
else:
|
||||||
|
if os.getenv('TRAVIS') is not None:
|
||||||
|
_save_failed_test(image, std_image, standard_file)
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def assert_image_match(im1, im2, min_corr=0.9, px_threshold=50.,
|
||||||
|
px_count=None, max_px_diff=None, avg_px_diff=None,
|
||||||
|
img_diff=None):
|
||||||
|
"""Check that two images match.
|
||||||
|
|
||||||
|
Images that differ in shape or dtype will fail unconditionally.
|
||||||
|
Further tests for similarity depend on the arguments supplied.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
im1 : (h, w, 4) ndarray
|
||||||
|
Test output image
|
||||||
|
im2 : (h, w, 4) ndarray
|
||||||
|
Test standard image
|
||||||
|
min_corr : float or None
|
||||||
|
Minimum allowed correlation coefficient between corresponding image
|
||||||
|
values (see numpy.corrcoef)
|
||||||
|
px_threshold : float
|
||||||
|
Minimum value difference at which two pixels are considered different
|
||||||
|
px_count : int or None
|
||||||
|
Maximum number of pixels that may differ
|
||||||
|
max_px_diff : float or None
|
||||||
|
Maximum allowed difference between pixels
|
||||||
|
avg_px_diff : float or None
|
||||||
|
Average allowed difference between pixels
|
||||||
|
img_diff : float or None
|
||||||
|
Maximum allowed summed difference between images
|
||||||
|
|
||||||
|
"""
|
||||||
|
assert im1.ndim == 3
|
||||||
|
assert im1.shape[2] == 4
|
||||||
|
assert im1.dtype == im2.dtype
|
||||||
|
|
||||||
|
diff = im1.astype(float) - im2.astype(float)
|
||||||
|
if img_diff is not None:
|
||||||
|
assert np.abs(diff).sum() <= img_diff
|
||||||
|
|
||||||
|
pxdiff = diff.max(axis=2) # largest value difference per pixel
|
||||||
|
mask = np.abs(pxdiff) >= px_threshold
|
||||||
|
if px_count is not None:
|
||||||
|
assert mask.sum() <= px_count
|
||||||
|
|
||||||
|
masked_diff = diff[mask]
|
||||||
|
if max_px_diff is not None and masked_diff.size > 0:
|
||||||
|
assert masked_diff.max() <= max_px_diff
|
||||||
|
if avg_px_diff is not None and masked_diff.size > 0:
|
||||||
|
assert masked_diff.mean() <= avg_px_diff
|
||||||
|
|
||||||
|
if min_corr is not None:
|
||||||
|
with np.errstate(invalid='ignore'):
|
||||||
|
corr = np.corrcoef(im1.ravel(), im2.ravel())[0, 1]
|
||||||
|
assert corr >= min_corr
|
||||||
|
|
||||||
|
|
||||||
|
def _save_failed_test(data, expect, filename):
|
||||||
|
from ..io import _make_png
|
||||||
|
commit, error = run_subprocess(['git', 'rev-parse', 'HEAD'])
|
||||||
|
name = filename.split('/')
|
||||||
|
name.insert(-1, commit.strip())
|
||||||
|
filename = '/'.join(name)
|
||||||
|
host = 'data.pyqtgraph.org'
|
||||||
|
|
||||||
|
# concatenate data, expect, and diff into a single image
|
||||||
|
ds = data.shape
|
||||||
|
es = expect.shape
|
||||||
|
|
||||||
|
shape = (max(ds[0], es[0]) + 4, ds[1] + es[1] + 8 + max(ds[1], es[1]), 4)
|
||||||
|
img = np.empty(shape, dtype=np.ubyte)
|
||||||
|
img[..., :3] = 100
|
||||||
|
img[..., 3] = 255
|
||||||
|
|
||||||
|
img[2:2+ds[0], 2:2+ds[1], :ds[2]] = data
|
||||||
|
img[2:2+es[0], ds[1]+4:ds[1]+4+es[1], :es[2]] = expect
|
||||||
|
|
||||||
|
diff = make_diff_image(data, expect)
|
||||||
|
img[2:2+diff.shape[0], -diff.shape[1]-2:-2] = diff
|
||||||
|
|
||||||
|
png = _make_png(img)
|
||||||
|
conn = httplib.HTTPConnection(host)
|
||||||
|
req = urllib.urlencode({'name': filename,
|
||||||
|
'data': base64.b64encode(png)})
|
||||||
|
conn.request('POST', '/upload.py', req)
|
||||||
|
response = conn.getresponse().read()
|
||||||
|
conn.close()
|
||||||
|
print("\nImage comparison failed. Test result: %s %s Expected result: "
|
||||||
|
"%s %s" % (data.shape, data.dtype, expect.shape, expect.dtype))
|
||||||
|
print("Uploaded to: \nhttp://%s/data/%s" % (host, filename))
|
||||||
|
if not response.startswith(b'OK'):
|
||||||
|
print("WARNING: Error uploading data to %s" % host)
|
||||||
|
print(response)
|
||||||
|
|
||||||
|
|
||||||
|
def make_diff_image(im1, im2):
|
||||||
|
"""Return image array showing the differences between im1 and im2.
|
||||||
|
|
||||||
|
Handles images of different shape. Alpha channels are not compared.
|
||||||
|
"""
|
||||||
|
ds = im1.shape
|
||||||
|
es = im2.shape
|
||||||
|
|
||||||
|
diff = np.empty((max(ds[0], es[0]), max(ds[1], es[1]), 4), dtype=int)
|
||||||
|
diff[..., :3] = 128
|
||||||
|
diff[..., 3] = 255
|
||||||
|
diff[:ds[0], :ds[1], :min(ds[2], 3)] += im1[..., :3]
|
||||||
|
diff[:es[0], :es[1], :min(es[2], 3)] -= im2[..., :3]
|
||||||
|
diff = np.clip(diff, 0, 255).astype(np.ubyte)
|
||||||
|
return diff
|
||||||
|
|
||||||
|
|
||||||
|
class ImageTester(QtGui.QWidget):
|
||||||
|
"""Graphical interface for auditing image comparison tests.
|
||||||
|
"""
|
||||||
|
def __init__(self):
|
||||||
|
self.lastKey = None
|
||||||
|
|
||||||
|
QtGui.QWidget.__init__(self)
|
||||||
|
|
||||||
|
layout = QtGui.QGridLayout()
|
||||||
|
self.setLayout(self.layout)
|
||||||
|
|
||||||
|
view = GraphicsLayoutWidget()
|
||||||
|
self.layout.addWidget(view, 0, 0, 1, 2)
|
||||||
|
|
||||||
|
self.label = QtGui.QLabel()
|
||||||
|
self.layout.addWidget(self.label, 1, 0, 1, 2)
|
||||||
|
|
||||||
|
#self.passBtn = QtGui.QPushButton('Pass')
|
||||||
|
#self.failBtn = QtGui.QPushButton('Fail')
|
||||||
|
#self.layout.addWidget(self.passBtn, 2, 0)
|
||||||
|
#self.layout.addWidget(self.failBtn, 2, 0)
|
||||||
|
|
||||||
|
self.views = (self.view.addViewBox(row=0, col=0),
|
||||||
|
self.view.addViewBox(row=0, col=1),
|
||||||
|
self.view.addViewBox(row=0, col=2))
|
||||||
|
labelText = ['test output', 'standard', 'diff']
|
||||||
|
for i, v in enumerate(self.views):
|
||||||
|
v.setAspectLocked(1)
|
||||||
|
v.invertY()
|
||||||
|
v.image = ImageItem()
|
||||||
|
v.addItem(v.image)
|
||||||
|
v.label = TextItem(labelText[i])
|
||||||
|
|
||||||
|
self.views[1].setXLink(self.views[0])
|
||||||
|
self.views[2].setXLink(self.views[0])
|
||||||
|
|
||||||
|
def test(self, im1, im2, message):
|
||||||
|
self.show()
|
||||||
|
if im2 is None:
|
||||||
|
message += 'Image1: %s %s Image2: [no standard]' % (im1.shape, im1.dtype)
|
||||||
|
im2 = np.zeros((1, 1, 3), dtype=np.ubyte)
|
||||||
|
else:
|
||||||
|
message += 'Image1: %s %s Image2: %s %s' % (im1.shape, im1.dtype, im2.shape, im2.dtype)
|
||||||
|
self.label.setText(message)
|
||||||
|
|
||||||
|
self.views[0].image.setImage(im1)
|
||||||
|
self.views[1].image.setImage(im2)
|
||||||
|
diff = make_diff_image(im1, im2)
|
||||||
|
|
||||||
|
self.views[2].image.setImage(diff)
|
||||||
|
self.views[0].autoRange()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
self.app.process_events()
|
||||||
|
lastKey = self.lastKey
|
||||||
|
self.lastKey = None
|
||||||
|
if lastKey is None:
|
||||||
|
pass
|
||||||
|
elif lastKey.lower() == 'p':
|
||||||
|
break
|
||||||
|
elif lastKey.lower() in ('f', 'esc'):
|
||||||
|
raise Exception("User rejected test result.")
|
||||||
|
time.sleep(0.03)
|
||||||
|
|
||||||
|
for v in self.views:
|
||||||
|
v.image.setImage(np.zeros((1, 1, 3), dtype=np.ubyte))
|
||||||
|
|
||||||
|
def keyPressEvent(self, event):
|
||||||
|
self.lastKey = event.text()
|
||||||
|
|
||||||
|
|
||||||
|
def get_test_data_repo():
|
||||||
|
"""Return the path to a git repository with the required commit checked
|
||||||
|
out.
|
||||||
|
|
||||||
|
If the repository does not exist, then it is cloned from
|
||||||
|
https://github.com/vispy/test-data. If the repository already exists
|
||||||
|
then the required commit is checked out.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# This tag marks the test-data commit that this version of vispy should
|
||||||
|
# be tested against. When adding or changing test images, create
|
||||||
|
# and push a new tag and update this variable.
|
||||||
|
test_data_tag = 'test-data-4'
|
||||||
|
|
||||||
|
data_path = config['test_data_path']
|
||||||
|
git_path = 'https://github.com/pyqtgraph/test-data'
|
||||||
|
gitbase = git_cmd_base(data_path)
|
||||||
|
|
||||||
|
if os.path.isdir(data_path):
|
||||||
|
# Already have a test-data repository to work with.
|
||||||
|
|
||||||
|
# Get the commit ID of test_data_tag. Do a fetch if necessary.
|
||||||
|
try:
|
||||||
|
tag_commit = git_commit_id(data_path, test_data_tag)
|
||||||
|
except NameError:
|
||||||
|
cmd = gitbase + ['fetch', '--tags', 'origin']
|
||||||
|
print(' '.join(cmd))
|
||||||
|
check_call(cmd)
|
||||||
|
try:
|
||||||
|
tag_commit = git_commit_id(data_path, test_data_tag)
|
||||||
|
except NameError:
|
||||||
|
raise Exception("Could not find tag '%s' in test-data repo at"
|
||||||
|
" %s" % (test_data_tag, data_path))
|
||||||
|
except Exception:
|
||||||
|
if not os.path.exists(os.path.join(data_path, '.git')):
|
||||||
|
raise Exception("Directory '%s' does not appear to be a git "
|
||||||
|
"repository. Please remove this directory." %
|
||||||
|
data_path)
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
# If HEAD is not the correct commit, then do a checkout
|
||||||
|
if git_commit_id(data_path, 'HEAD') != tag_commit:
|
||||||
|
print("Checking out test-data tag '%s'" % test_data_tag)
|
||||||
|
check_call(gitbase + ['checkout', test_data_tag])
|
||||||
|
|
||||||
|
else:
|
||||||
|
print("Attempting to create git clone of test data repo in %s.." %
|
||||||
|
data_path)
|
||||||
|
|
||||||
|
parent_path = os.path.split(data_path)[0]
|
||||||
|
if not os.path.isdir(parent_path):
|
||||||
|
os.makedirs(parent_path)
|
||||||
|
|
||||||
|
if os.getenv('TRAVIS') is not None:
|
||||||
|
# Create a shallow clone of the test-data repository (to avoid
|
||||||
|
# downloading more data than is necessary)
|
||||||
|
os.makedirs(data_path)
|
||||||
|
cmds = [
|
||||||
|
gitbase + ['init'],
|
||||||
|
gitbase + ['remote', 'add', 'origin', git_path],
|
||||||
|
gitbase + ['fetch', '--tags', 'origin', test_data_tag,
|
||||||
|
'--depth=1'],
|
||||||
|
gitbase + ['checkout', '-b', 'master', 'FETCH_HEAD'],
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
# Create a full clone
|
||||||
|
cmds = [['git', 'clone', git_path, data_path]]
|
||||||
|
|
||||||
|
for cmd in cmds:
|
||||||
|
print(' '.join(cmd))
|
||||||
|
rval = check_call(cmd)
|
||||||
|
if rval == 0:
|
||||||
|
continue
|
||||||
|
raise RuntimeError("Test data path '%s' does not exist and could "
|
||||||
|
"not be created with git. Either create a git "
|
||||||
|
"clone of %s or set the test_data_path "
|
||||||
|
"variable to an existing clone." %
|
||||||
|
(data_path, git_path))
|
||||||
|
|
||||||
|
return data_path
|
||||||
|
|
||||||
|
|
||||||
|
def git_cmd_base(path):
|
||||||
|
return ['git', '--git-dir=%s/.git' % path, '--work-tree=%s' % path]
|
||||||
|
|
||||||
|
|
||||||
|
def git_status(path):
|
||||||
|
"""Return a string listing all changes to the working tree in a git
|
||||||
|
repository.
|
||||||
|
"""
|
||||||
|
cmd = git_cmd_base(path) + ['status', '--porcelain']
|
||||||
|
return run_subprocess(cmd, stderr=None, universal_newlines=True)[0]
|
||||||
|
|
||||||
|
|
||||||
|
def git_commit_id(path, ref):
|
||||||
|
"""Return the commit id of *ref* in the git repository at *path*.
|
||||||
|
"""
|
||||||
|
cmd = git_cmd_base(path) + ['show', ref]
|
||||||
|
try:
|
||||||
|
output = run_subprocess(cmd, stderr=None, universal_newlines=True)[0]
|
||||||
|
except CalledProcessError:
|
||||||
|
raise NameError("Unknown git reference '%s'" % ref)
|
||||||
|
commit = output.split('\n')[0]
|
||||||
|
assert commit[:7] == 'commit '
|
||||||
|
return commit[7:]
|
Loading…
Reference in New Issue
Block a user