Differential D3064 Diff 10024 tests/python/render_report.py

Changeset View

Standalone View

tests/python/render_report.py

This file was added.

Property	Old Value	New Value
File Mode	null	100755

				#!/usr/bin/env python3
				campbellbartonUnsubmitted Done Inline Actions We could place this in a `modules/` subdir to keep all tests executable tests. campbellbarton: We could place this in a `modules/` subdir to keep all tests executable tests.
				# Apache License, Version 2.0

				import glob
				import os
				import pathlib
				import shutil
				import subprocess
				import sys
				import time


				class COLORS_ANSI:
				RED = '\033[00;31m'
				GREEN = '\033[00;32m'
				ENDC = '\033[0m'


				class COLORS_DUMMY:
				RED = ''
				GREEN = ''
				ENDC = ''

				COLORS = COLORS_DUMMY


				def print_message(message, type=None, status=''):
				if type == 'SUCCESS':
				print(COLORS.GREEN, end="")
				elif type == 'FAILURE':
				print(COLORS.RED, end="")
				status_text = ...
				if status == 'RUN':
				status_text = " RUN "
				elif status == 'OK':
				status_text = " OK "
				elif status == 'PASSED':
				status_text = " PASSED "
				elif status == 'FAILED':
				status_text = " FAILED "
				else:
				status_text = status
				if status_text:
				print("[{}]" . format(status_text), end="")
				print(COLORS.ENDC, end="")
				print(" {}" . format(message))
				sys.stdout.flush()


				def blend_list(path):
				for dirpath, dirnames, filenames in os.walk(path):
				for filename in filenames:
				if filename.lower().endswith(".blend"):
				filepath = os.path.join(dirpath, filename)
				yield filepath

				def test_get_name(filepath):
				filename = os.path.basename(filepath)
				return os.path.splitext(filename)[0]

				def test_get_images(output_dir, filepath):
				testname = test_get_name(filepath)
				dirpath = os.path.dirname(filepath)

				old_dirpath = os.path.join(dirpath, "reference_renders")
				old_img = os.path.join(old_dirpath, testname + ".png")

				ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref")
				ref_img = os.path.join(ref_dirpath, testname + ".png")
				if not os.path.exists(ref_dirpath):
				os.makedirs(ref_dirpath)
				if os.path.exists(old_img):
				shutil.copy(old_img, ref_img)

				new_dirpath = os.path.join(output_dir, os.path.basename(dirpath))
				if not os.path.exists(new_dirpath):
				os.makedirs(new_dirpath)
				new_img = os.path.join(new_dirpath, testname + ".png")

				diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff")
				if not os.path.exists(diff_dirpath):
				os.makedirs(diff_dirpath)
				diff_img = os.path.join(diff_dirpath, testname + ".diff.png")

				return old_img, ref_img, new_img, diff_img


				class Report:
				campbellbartonUnsubmitted Done Inline Actions picky best to use `__slots__`, main advantage is typos in assignments do go unnoticed. campbellbarton: picky best to use `__slots__`, main advantage is typos in assignments do go unnoticed.
				def __init__(self, name, output_dir, idiff):
				self.name = name
				self.output_dir = output_dir
				self.idiff = idiff

				self.pixelated = False
				self.verbose = os.environ.get("BLENDER_VERBOSE") is not None
				self.update = os.getenv('BLENDER_TEST_UPDATE') is not None

				if os.environ.get("BLENDER_TEST_COLOR") is not None:
				global COLORS, COLORS_ANSI
				COLORS = COLORS_ANSI

				self.failed_tests = ""
				self.passed_tests = ""

				if not os.path.exists(output_dir):
				os.makedirs(output_dir)

				def set_pixelated(self, pixelated):
				self.pixelated = pixelated

				def run(self, dirpath, render_cb):
				# Run tests and output report
				dirname = os.path.basename(dirpath)
				ok = self._run_all_tests(dirname, dirpath, render_cb)
				self._write_html(dirname)
				return ok

				def _write_html(self, dirname):
				# write intermediate data for single test
				outdir = os.path.join(self.output_dir, dirname)
				if not os.path.exists(outdir):
				os.makedirs(outdir)

				filepath = os.path.join(outdir, "failed.data")
				pathlib.Path(filepath).write_text(self.failed_tests)

				filepath = os.path.join(outdir, "passed.data")
				pathlib.Path(filepath).write_text(self.passed_tests)

				# gather intermediate data for all tests
				failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data")))
				passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data")))

				failed_tests = ""
				passed_tests = ""

				for filename in failed_data:
				filepath = os.path.join(self.output_dir, filename)
				failed_tests += pathlib.Path(filepath).read_text()
				for filename in passed_data:
				filepath = os.path.join(self.output_dir, filename)
				passed_tests += pathlib.Path(filepath).read_text()

				if self.pixelated:
				image_rendering = 'pixelated'
				else:
				image_rendering = 'auto'

				if len(failed_tests) > 0:
				message = "<p>Run <tt>BLENDER_TEST_UPDATE=1 ctest</tt> to create or update reference images for failed tests.</p>"
				else:
				message = ""

				# write html for all tests
				self.html = """
				<html>
				<head>
				<title>{}</title>
				<style>
				img {{ image-rendering: {}; width: 256px; background-color: #000; }}
				img.render {{
				background-color: #fff;
				background-image:
				-moz-linear-gradient(45deg, #eee 25%, transparent 25%),
				-moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
				-moz-linear-gradient(45deg, transparent 75%, #eee 75%),
				-moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
				background-image:
				-webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
				-webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
				-webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
				-webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));

				-moz-background-size:50px 50px;
				background-size:50px 50px;
				-webkit-background-size:50px 51px; /* override value for shitty webkit */

				background-position:0 0, 25px 0, 25px -25px, 0px 25px;
				}}
				table td:first-child {{ width: 256px; }}
				</style>
				<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
				</head>
				<body>
				<div class="container">
				<br/>
				<h1>{}</h1>
				{}
				<br/>
				<table class="table table-striped">
				<thead class="thead-default">
				<tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
				</thead>
				{}{}
				</table>
				<br/>
				</div>
				</body>
				</html>
				""" . format(self.name, image_rendering, self.name, message, failed_tests, passed_tests)
				campbellbartonUnsubmitted Done Inline Actions picky realize this is just moved over from previous code, would use named args here: """blah blah {name} ... blah blah {message} """.format(name=self.name, message=message) campbellbarton: picky realize this is just moved over from previous code, would use named args here…

				filepath = os.path.join(self.output_dir, "report.html")
				pathlib.Path(filepath).write_text(self.html)

				print_message("Report saved to: " + pathlib.Path(filepath).as_uri())

				def _relative_url(self, filepath):
				relpath = os.path.relpath(filepath, self.output_dir)
				return pathlib.Path(relpath).as_posix()

				def _write_test_html(self, testname, filepath, error):
				name = test_get_name(filepath)
				name = name.replace('_', ' ')

				old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath)

				status = error if error else ""
				style = """ style="background-color: #f99;" """ if error else ""

				new_url = self._relative_url(new_img)
				ref_url = self._relative_url(ref_img)
				diff_url = self._relative_url(diff_img)

				test_html = """
				<tr{}>
				<td><b>{}</b><br/>{}<br/>{}</td>
				<td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
				<td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
				<td><img src="{}"></td>
				</tr>""" . format(style, name, testname, status,
				new_url, ref_url, new_url,
				ref_url, new_url, ref_url,
				diff_url)

				if error:
				self.failed_tests += test_html
				else:
				self.passed_tests += test_html


				def _diff_output(self, filepath, tmp_filepath):
				old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath)

				# create reference render directory
				old_dirpath = os.path.dirname(old_img)
				if not os.path.exists(old_dirpath):
				os.makedirs(old_dirpath)

				# copy new image
				if os.path.exists(new_img):
				os.remove(new_img)
				if os.path.exists(tmp_filepath):
				shutil.copy(tmp_filepath, new_img)

				if os.path.exists(ref_img):
				# diff test with threshold
				command = (
				self.idiff,
				"-fail", "0.016",
				"-failpercent", "1",
				ref_img,
				tmp_filepath,
				)
				try:
				subprocess.check_output(command)
				failed = False
				except subprocess.CalledProcessError as e:
				if self.verbose:
				print_message(e.output.decode("utf-8"))
				failed = e.returncode != 1
				else:
				if not self.update:
				return False

				failed = True

				if failed and self.update:
				# update reference
				shutil.copy(new_img, ref_img)
				shutil.copy(new_img, old_img)
				failed = False

				# generate diff image
				command = (
				self.idiff,
				"-o", diff_img,
				"-abs", "-scale", "16",
				ref_img,
				tmp_filepath
				)

				try:
				subprocess.check_output(command)
				except subprocess.CalledProcessError as e:
				if self.verbose:
				print_message(e.output.decode("utf-8"))

				return not failed


				def _run_test(self, filepath, render_cb):
				testname = test_get_name(filepath)
				print_message(testname, 'SUCCESS', 'RUN')
				time_start = time.time()
				tmp_filepath = os.path.join(self.output_dir, "tmp")

				error = render_cb(filepath, tmp_filepath)
				status = "FAIL"
				if not error:
				if not self._diff_output(filepath, tmp_filepath):
				error = "VERIFY"

				if os.path.exists(tmp_filepath):
				os.remove(tmp_filepath)

				time_end = time.time()
				elapsed_ms = int((time_end - time_start) * 1000)
				if not error:
				print_message("{} ({} ms)" . format(testname, elapsed_ms),
				'SUCCESS', 'OK')
				else:
				if error == "NO_ENGINE":
				print_message("Can't perform tests because the render engine failed to load!")
				return error
				elif error == "NO_START":
				print_message('Can not perform tests because blender fails to start.',
				'Make sure INSTALL target was run.')
				return error
				elif error == 'VERIFY':
				print_message("Rendered result is different from reference image")
				else:
				print_message("Unknown error %r" % error)
				print_message("{} ({} ms)" . format(testname, elapsed_ms),
				'FAILURE', 'FAILED')
				return error


				def _run_all_tests(self, dirname, dirpath, render_cb):
				passed_tests = []
				failed_tests = []
				all_files = list(blend_list(dirpath))
				all_files.sort()
				print_message("Running {} tests from 1 test case." .
				format(len(all_files)),
				'SUCCESS', "==========")
				time_start = time.time()
				for filepath in all_files:
				error = self._run_test(filepath, render_cb)
				testname = test_get_name(filepath)
				if error:
				if error == "NO_ENGINE":
				return False
				elif error == "NO_START":
				return False
				failed_tests.append(testname)
				else:
				passed_tests.append(testname)
				self._write_test_html(dirname, filepath, error)
				time_end = time.time()
				elapsed_ms = int((time_end - time_start) * 1000)
				print_message("")
				print_message("{} tests from 1 test case ran. ({} ms total)" .
				format(len(all_files), elapsed_ms),
				'SUCCESS', "==========")
				print_message("{} tests." .
				format(len(passed_tests)),
				'SUCCESS', 'PASSED')
				if failed_tests:
				print_message("{} tests, listed below:" .
				format(len(failed_tests)),
				'FAILURE', 'FAILED')
				failed_tests.sort()
				for test in failed_tests:
				print_message("{}" . format(test), 'FAILURE', "FAILED")

				return not bool(failed_tests)