Page MenuHome
Differential D3064 Diff 10024 tests/python/cycles_render_tests.py

Changeset View

Standalone View

View Options

tests/python/cycles_render_tests.py

#!/usr/bin/env python3 #!/usr/bin/env python3
# Apache License, Version 2.0 # Apache License, Version 2.0
import argparse import argparse
import glob
import os import os
import pathlib
import shlex import shlex
import shutil import shutil
import subprocess import subprocess
import sys import sys
import time
import tempfile
class COLORS_ANSI: def render_file(filepath, output_filepath):
RED = '\033[00;31m'
GREEN = '\033[00;32m'
ENDC = '\033[0m'
class COLORS_DUMMY:
RED = ''
GREEN = ''
ENDC = ''
COLORS = COLORS_DUMMY
def print_message(message, type=None, status=''):
if type == 'SUCCESS':
print(COLORS.GREEN, end="")
elif type == 'FAILURE':
print(COLORS.RED, end="")
status_text = ...
if status == 'RUN':
status_text = " RUN "
elif status == 'OK':
status_text = " OK "
elif status == 'PASSED':
status_text = " PASSED "
elif status == 'FAILED':
status_text = " FAILED "
else:
status_text = status
if status_text:
print("[{}]" . format(status_text), end="")
print(COLORS.ENDC, end="")
print(" {}" . format(message))
sys.stdout.flush()
def render_file(filepath):
dirname = os.path.dirname(filepath) dirname = os.path.dirname(filepath)
basedir = os.path.dirname(dirname) basedir = os.path.dirname(dirname)
subject = os.path.basename(dirname) subject = os.path.basename(dirname)
custom_args = os.getenv('CYCLESTEST_ARGS') custom_args = os.getenv('CYCLESTEST_ARGS')
custom_args = shlex.split(custom_args) if custom_args else [] custom_args = shlex.split(custom_args) if custom_args else []
# OSL and GPU examples # OSL and GPU examples
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
# custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
frame_filepath = output_filepath + '0001.png'
if subject == 'opengl': if subject == 'opengl':
command = [ command = [
BLENDER, BLENDER,
"--window-geometry", "0", "0", "1", "1", "--window-geometry", "0", "0", "1", "1",
"-noaudio", "-noaudio",
"--factory-startup", "--factory-startup",
"--enable-autoexec", "--enable-autoexec",
filepath, filepath,
"-E", "CYCLES"] "-E", "CYCLES"]
command += custom_args command += custom_args
command += [ command += [
"-o", TEMP_FILE_MASK, "-o", output_filepath,
"-F", "PNG", "-F", "PNG",
'--python', os.path.join(basedir, '--python', os.path.join(basedir,
"util", "util",
"render_opengl.py")] "render_opengl.py")]
elif subject == 'bake': elif subject == 'bake':
command = [ command = [
BLENDER, BLENDER,
"-b", "-b",
"-noaudio", "-noaudio",
"--factory-startup", "--factory-startup",
"--enable-autoexec", "--enable-autoexec",
filepath, filepath,
"-E", "CYCLES"] "-E", "CYCLES"]
command += custom_args command += custom_args
command += [ command += [
"-o", TEMP_FILE_MASK, "-o", output_filepath,
"-F", "PNG", "-F", "PNG",
'--python', os.path.join(basedir, '--python', os.path.join(basedir,
"util", "util",
"render_bake.py")] "render_bake.py")]
else: else:
command = [ command = [
BLENDER, BLENDER,
"--background", "--background",
"-noaudio", "-noaudio",
"--factory-startup", "--factory-startup",
"--enable-autoexec", "--enable-autoexec",
filepath, filepath,
"-E", "CYCLES"] "-E", "CYCLES"]
command += custom_args command += custom_args
command += [ command += [
"-o", TEMP_FILE_MASK, "-o", output_filepath,
"-F", "PNG", "-F", "PNG",
"-f", "1"] "-f", "1"]
try: try:
# Success
output = subprocess.check_output(command) output = subprocess.check_output(command)
if os.path.exists(frame_filepath):
shutil.copy(frame_filepath, output_filepath)
os.remove(frame_filepath)
if VERBOSE: if VERBOSE:
print(output.decode("utf-8")) print(output.decode("utf-8"))
return None return None
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
if os.path.exists(TEMP_FILE): # Error
os.remove(TEMP_FILE) if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE: if VERBOSE:
print(e.output.decode("utf-8")) print(e.output.decode("utf-8"))
if b"Error: engine not found" in e.output: if b"Error: engine not found" in e.output:
return "NO_CYCLES" return "NO_ENGINE"
elif b"blender probably wont start" in e.output: elif b"blender probably wont start" in e.output:
return "NO_START" return "NO_START"
return "CRASH" return "CRASH"
except BaseException as e: except BaseException as e:
if os.path.exists(TEMP_FILE): # Crash
os.remove(TEMP_FILE) if os.path.exists(frame_filepath):
os.remove(frame_filepath)
if VERBOSE: if VERBOSE:
print(e) print(e)
return "CRASH" return "CRASH"
def test_get_name(filepath):
filename = os.path.basename(filepath)
return os.path.splitext(filename)[0]
def test_get_images(filepath):
testname = test_get_name(filepath)
dirpath = os.path.dirname(filepath)
old_dirpath = os.path.join(dirpath, "reference_renders")
old_img = os.path.join(old_dirpath, testname + ".png")
ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref")
ref_img = os.path.join(ref_dirpath, testname + ".png")
if not os.path.exists(ref_dirpath):
os.makedirs(ref_dirpath)
if os.path.exists(old_img):
shutil.copy(old_img, ref_img)
new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath))
if not os.path.exists(new_dirpath):
os.makedirs(new_dirpath)
new_img = os.path.join(new_dirpath, testname + ".png")
diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff")
if not os.path.exists(diff_dirpath):
os.makedirs(diff_dirpath)
diff_img = os.path.join(diff_dirpath, testname + ".diff.png")
return old_img, ref_img, new_img, diff_img
class Report:
def __init__(self, testname):
self.failed_tests = ""
self.passed_tests = ""
self.testname = testname
def output(self):
# write intermediate data for single test
outdir = os.path.join(OUTDIR, self.testname)
if not os.path.exists(outdir):
os.makedirs(outdir)
filepath = os.path.join(outdir, "failed.data")
pathlib.Path(filepath).write_text(self.failed_tests)
filepath = os.path.join(outdir, "passed.data")
pathlib.Path(filepath).write_text(self.passed_tests)
# gather intermediate data for all tests
failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data")))
passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data")))
failed_tests = ""
passed_tests = ""
for filename in failed_data:
filepath = os.path.join(OUTDIR, filename)
failed_tests += pathlib.Path(filepath).read_text()
for filename in passed_data:
filepath = os.path.join(OUTDIR, filename)
passed_tests += pathlib.Path(filepath).read_text()
# write html for all tests
self.html = """
<html>
<head>
<title>Cycles Test Report</title>
<style>
img {{ image-rendering: pixelated; width: 256px; background-color: #000; }}
img.render {{
background-color: #fff;
background-image:
-moz-linear-gradient(45deg, #eee 25%, transparent 25%),
-moz-linear-gradient(-45deg, #eee 25%, transparent 25%),
-moz-linear-gradient(45deg, transparent 75%, #eee 75%),
-moz-linear-gradient(-45deg, transparent 75%, #eee 75%);
background-image:
-webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)),
-webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)),
-webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)),
-webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee));
-moz-background-size:50px 50px;
background-size:50px 50px;
-webkit-background-size:50px 51px; /* override value for shitty webkit */
background-position:0 0, 25px 0, 25px -25px, 0px 25px;
}}
table td:first-child {{ width: 256px; }}
</style>
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css">
</head>
<body>
<div class="container">
<br/>
<h1>Cycles Test Report</h1>
<br/>
<table class="table table-striped">
<thead class="thead-default">
<tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th>
</thead>
{}{}
</table>
<br/>
</div>
</body>
</html>
""" . format(failed_tests, passed_tests)
filepath = os.path.join(OUTDIR, "report.html")
pathlib.Path(filepath).write_text(self.html)
print_message("Report saved to: " + pathlib.Path(filepath).as_uri())
def relative_url(self, filepath):
relpath = os.path.relpath(filepath, OUTDIR)
return pathlib.Path(relpath).as_posix()
def add_test(self, filepath, error):
name = test_get_name(filepath)
name = name.replace('_', ' ')
old_img, ref_img, new_img, diff_img = test_get_images(filepath)
status = error if error else ""
style = """ style="background-color: #f99;" """ if error else ""
new_url = self.relative_url(new_img)
ref_url = self.relative_url(ref_img)
diff_url = self.relative_url(diff_img)
test_html = """
<tr{}>
<td><b>{}</b><br/>{}<br/>{}</td>
<td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
<td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td>
<td><img src="{}"></td>
</tr>""" . format(style, name, self.testname, status,
new_url, ref_url, new_url,
ref_url, new_url, ref_url,
diff_url)
if error:
self.failed_tests += test_html
else:
self.passed_tests += test_html
def verify_output(report, filepath):
old_img, ref_img, new_img, diff_img = test_get_images(filepath)
# copy new image
if os.path.exists(new_img):
os.remove(new_img)
if os.path.exists(TEMP_FILE):
shutil.copy(TEMP_FILE, new_img)
update = os.getenv('CYCLESTEST_UPDATE')
if os.path.exists(ref_img):
# diff test with threshold
command = (
IDIFF,
"-fail", "0.016",
"-failpercent", "1",
ref_img,
TEMP_FILE,
)
try:
subprocess.check_output(command)
failed = False
except subprocess.CalledProcessError as e:
if VERBOSE:
print_message(e.output.decode("utf-8"))
failed = e.returncode != 1
else:
if not update:
return False
failed = True
if failed and update:
# update reference
shutil.copy(new_img, ref_img)
shutil.copy(new_img, old_img)
failed = False
# generate diff image
command = (
IDIFF,
"-o", diff_img,
"-abs", "-scale", "16",
ref_img,
TEMP_FILE
)
try:
subprocess.check_output(command)
except subprocess.CalledProcessError as e:
if VERBOSE:
print_message(e.output.decode("utf-8"))
return not failed
def run_test(report, filepath):
testname = test_get_name(filepath)
spacer = "." * (32 - len(testname))
print_message(testname, 'SUCCESS', 'RUN')
time_start = time.time()
error = render_file(filepath)
status = "FAIL"
if not error:
if not verify_output(report, filepath):
error = "VERIFY"
time_end = time.time()
elapsed_ms = int((time_end - time_start) * 1000)
if not error:
print_message("{} ({} ms)" . format(testname, elapsed_ms),
'SUCCESS', 'OK')
else:
if error == "NO_CYCLES":
print_message("Can't perform tests because Cycles failed to load!")
return error
elif error == "NO_START":
print_message('Can not perform tests because blender fails to start.',
'Make sure INSTALL target was run.')
return error
elif error == 'VERIFY':
print_message("Rendered result is different from reference image")
else:
print_message("Unknown error %r" % error)
print_message("{} ({} ms)" . format(testname, elapsed_ms),
'FAILURE', 'FAILED')
return error
def blend_list(path):
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
if filename.lower().endswith(".blend"):
filepath = os.path.join(dirpath, filename)
yield filepath
def run_all_tests(dirpath):
passed_tests = []
failed_tests = []
all_files = list(blend_list(dirpath))
all_files.sort()
report = Report(os.path.basename(dirpath))
print_message("Running {} tests from 1 test case." .
format(len(all_files)),
'SUCCESS', "==========")
time_start = time.time()
for filepath in all_files:
error = run_test(report, filepath)
testname = test_get_name(filepath)
if error:
if error == "NO_CYCLES":
return False
elif error == "NO_START":
return False
failed_tests.append(testname)
else:
passed_tests.append(testname)
report.add_test(filepath, error)
time_end = time.time()
elapsed_ms = int((time_end - time_start) * 1000)
print_message("")
print_message("{} tests from 1 test case ran. ({} ms total)" .
format(len(all_files), elapsed_ms),
'SUCCESS', "==========")
print_message("{} tests." .
format(len(passed_tests)),
'SUCCESS', 'PASSED')
if failed_tests:
print_message("{} tests, listed below:" .
format(len(failed_tests)),
'FAILURE', 'FAILED')
failed_tests.sort()
for test in failed_tests:
print_message("{}" . format(test), 'FAILURE', "FAILED")
report.output()
return not bool(failed_tests)
def create_argparse(): def create_argparse():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-blender", nargs="+") parser.add_argument("-blender", nargs="+")
parser.add_argument("-testdir", nargs=1) parser.add_argument("-testdir", nargs=1)
parser.add_argument("-outdir", nargs=1) parser.add_argument("-outdir", nargs=1)
parser.add_argument("-idiff", nargs=1) parser.add_argument("-idiff", nargs=1)
return parser return parser
def main(): def main():
parser = create_argparse() parser = create_argparse()
args = parser.parse_args() args = parser.parse_args()
global COLORS global BLENDER, VERBOSE
global BLENDER, TESTDIR, IDIFF, OUTDIR
global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT
global VERBOSE
if os.environ.get("CYCLESTEST_COLOR") is not None:
COLORS = COLORS_ANSI
BLENDER = args.blender[0] BLENDER = args.blender[0]
TESTDIR = args.testdir[0]
IDIFF = args.idiff[0]
OUTDIR = args.outdir[0]
if not os.path.exists(OUTDIR):
os.makedirs(OUTDIR)
TEMP = tempfile.mkdtemp()
TEMP_FILE_MASK = os.path.join(TEMP, "test")
TEMP_FILE = TEMP_FILE_MASK + "0001.png"
TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py")
VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
ok = run_all_tests(TESTDIR) test_dir = args.testdir[0]
idiff = args.idiff[0]
# Cleanup temp files and folders output_dir = args.outdir[0]
if os.path.exists(TEMP_FILE):
os.remove(TEMP_FILE) import render_report
os.rmdir(TEMP) report = render_report.Report("Cycles Test Report", output_dir, idiff)
report.set_pixelated(True)
ok = report.run(test_dir, render_file)
sys.exit(not ok) sys.exit(not ok)
if __name__ == "__main__": if __name__ == "__main__":
main() main()