From fda8731dabb599655e8754297d728b91b3281872 Mon Sep 17 00:00:00 2001 From: KIU Shueng Chuan Date: Tue, 2 Mar 2021 09:05:52 +0800 Subject: [PATCH] reduce memory usage during data generation random.normal() generates as float64 and gets converted to a smaller dtype. generating all the needed data in a single call thus uses a lot more memory than is necessary. this changes it such that smaller chunks are generated. data clipping is also changed to be in-place. the gaussian filtering which gave the video a washed-out look is also removed. this also contributed to data generation time. --- examples/VideoSpeedTest.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/examples/VideoSpeedTest.py b/examples/VideoSpeedTest.py index be05dba9..48b431dc 100644 --- a/examples/VideoSpeedTest.py +++ b/examples/VideoSpeedTest.py @@ -165,16 +165,22 @@ def mkData(): mx = 1.0 else: raise ValueError(f"unable to handle dtype: {cacheKey[0]}") - + + chan_shape = (width, height) if ui.rgbCheck.isChecked(): - data = xp.random.normal(size=(frames,width,height,3), loc=loc, scale=scale) - data = pg.gaussianFilter(data, (0, 6, 6, 0)) + frame_shape = chan_shape + (3,) else: - data = xp.random.normal(size=(frames,width,height), loc=loc, scale=scale) - data = pg.gaussianFilter(data, (0, 6, 6)) - if cacheKey[0] != 'float': - data = xp.clip(data, 0, mx) - data = data.astype(dt) + frame_shape = chan_shape + data = xp.empty((frames,) + frame_shape, dtype=dt) + view = data.reshape((-1,) + chan_shape) + for idx in range(view.shape[0]): + subdata = xp.random.normal(loc=loc, scale=scale, size=chan_shape) + # note: gaussian filtering has been removed as it slows down array + # creation greatly. + if cacheKey[0] != 'float': + xp.clip(subdata, 0, mx, out=subdata) + view[idx] = subdata + data[:, 10, 10:50] = mx data[:, 9:12, 48] = mx data[:, 8:13, 47] = mx