Changeset View
Changeset View
Standalone View
Standalone View
tests/python/modules/render_report.py
| Show First 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | for root, dirs, files in os.walk(dirpath): | ||||
| yield filepath | yield filepath | ||||
| def test_get_name(filepath): | def test_get_name(filepath): | ||||
| filename = os.path.basename(filepath) | filename = os.path.basename(filepath) | ||||
| return os.path.splitext(filename)[0] | return os.path.splitext(filename)[0] | ||||
| def test_get_images(output_dir, filepath, reference_dir): | def test_get_images(output_dir, filepath, reference_dir,kernel, allow_cb): | ||||
| testname = test_get_name(filepath) | testname = test_get_name(filepath) | ||||
| dirpath = os.path.dirname(filepath) | dirpath = os.path.dirname(filepath) | ||||
| if allow_cb(filepath, kernel) == "ownref": | |||||
| ref_suffix = "_" + kernel | |||||
| else: | |||||
| ref_suffix = "" | |||||
| old_dirpath = os.path.join(dirpath, reference_dir) | old_dirpath = os.path.join(dirpath, reference_dir) | ||||
| old_img = os.path.join(old_dirpath, testname + ".png") | old_img = os.path.join(old_dirpath, testname + ref_suffix + ".png") | ||||
| ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") | ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") | ||||
| ref_img = os.path.join(ref_dirpath, testname + ".png") | ref_img = os.path.join(ref_dirpath, testname + ref_suffix + ".png") | ||||
| os.makedirs(ref_dirpath, exist_ok=True) | os.makedirs(ref_dirpath, exist_ok=True) | ||||
| if os.path.exists(old_img): | if os.path.exists(old_img): | ||||
| shutil.copy(old_img, ref_img) | shutil.copy(old_img, ref_img) | ||||
| new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) | new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) | ||||
| os.makedirs(new_dirpath, exist_ok=True) | os.makedirs(new_dirpath, exist_ok=True) | ||||
| new_img = os.path.join(new_dirpath, testname + ".png") | new_img = os.path.join(new_dirpath, testname + "_" + kernel + ".png") | ||||
| diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") | diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") | ||||
| os.makedirs(diff_dirpath, exist_ok=True) | os.makedirs(diff_dirpath, exist_ok=True) | ||||
| diff_img = os.path.join(diff_dirpath, testname + ".diff.png") | diff_img = os.path.join(diff_dirpath, testname + "_" + kernel + ".diff.png") | ||||
| return old_img, ref_img, new_img, diff_img | return old_img, ref_img, new_img, diff_img | ||||
| class Report: | class Report: | ||||
| __slots__ = ( | __slots__ = ( | ||||
| 'title', | 'title', | ||||
| 'output_dir', | 'output_dir', | ||||
| 'reference_dir', | 'reference_dir', | ||||
| Show All 32 Lines | def set_pixelated(self, pixelated): | ||||
| self.pixelated = pixelated | self.pixelated = pixelated | ||||
| def set_reference_dir(self, reference_dir): | def set_reference_dir(self, reference_dir): | ||||
| self.reference_dir = reference_dir | self.reference_dir = reference_dir | ||||
| def set_compare_engines(self, engine, other_engine): | def set_compare_engines(self, engine, other_engine): | ||||
| self.compare_engines = (engine, other_engine) | self.compare_engines = (engine, other_engine) | ||||
| def run(self, dirpath, blender, arguments_cb, batch=False): | def run(self, dirpath, blender, arguments_cb, allow_cb, batch=False, enabled_kernels=["host"]): | ||||
| # Run tests and output report. | # Run tests and output report. | ||||
| ok = True | |||||
| for kernel in enabled_kernels: | |||||
| dirname = os.path.basename(dirpath) | dirname = os.path.basename(dirpath) | ||||
| ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch) | test_result = self._run_all_tests(dirname, dirpath, blender, arguments_cb, allow_cb, batch,kernel) | ||||
| self._write_data(dirname) | self._write_data(dirname) | ||||
| self._write_html() | self._write_html() | ||||
| if self.compare_engines: | if self.compare_engines: | ||||
| self._write_html(comparison=True) | self._write_html(comparison=True) | ||||
| ok = ok and test_result | |||||
| return ok | return ok | ||||
| def _write_data(self, dirname): | def _write_data(self, dirname): | ||||
| # Write intermediate data for single test. | # Write intermediate data for single test. | ||||
| outdir = os.path.join(self.output_dir, dirname) | outdir = os.path.join(self.output_dir, dirname) | ||||
| os.makedirs(outdir, exist_ok=True) | os.makedirs(outdir, exist_ok=True) | ||||
| filepath = os.path.join(outdir, "failed.data") | filepath = os.path.join(outdir, "failed.data") | ||||
| ▲ Show 20 Lines • Show All 134 Lines • ▼ Show 20 Lines | </html> | ||||
| global_output_dir = os.path.dirname(self.output_dir) | global_output_dir = os.path.dirname(self.output_dir) | ||||
| global_failed = failed if not comparison else None | global_failed = failed if not comparison else None | ||||
| global_report.add(global_output_dir, "Render", self.title, filepath, global_failed) | global_report.add(global_output_dir, "Render", self.title, filepath, global_failed) | ||||
| def _relative_url(self, filepath): | def _relative_url(self, filepath): | ||||
| relpath = os.path.relpath(filepath, self.output_dir) | relpath = os.path.relpath(filepath, self.output_dir) | ||||
| return pathlib.Path(relpath).as_posix() | return pathlib.Path(relpath).as_posix() | ||||
| def _write_test_html(self, testname, filepath, error): | def _write_test_html(self, testname, filepath, error,kernel, allow_cb): | ||||
| name = test_get_name(filepath) | name = test_get_name(filepath) | ||||
| name = name.replace('_', ' ') | name = name.replace('_', ' ') | ||||
| old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir) | old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir,kernel,allow_cb) | ||||
| status = error if error else "" | status = error if error else "" | ||||
| tr_style = """ class="table-danger" """ if error else "" | tr_style = """ class="table-danger" """ if error else "" | ||||
| new_url = self._relative_url(new_img) | new_url = self._relative_url(new_img) | ||||
| ref_url = self._relative_url(ref_img) | ref_url = self._relative_url(ref_img) | ||||
| diff_url = self._relative_url(diff_img) | diff_url = self._relative_url(diff_img) | ||||
| test_html = """ | test_html = """ | ||||
| <tr{tr_style}> | <tr{tr_style}> | ||||
| <td><b>{name}</b><br/>{testname}<br/>{status}</td> | <td><b>{name}</b><br/>{testname} ({kernel})<br/>{status}</td> | ||||
| <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td> | <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td> | ||||
| <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td> | <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td> | ||||
| <td><img src="{diff_url}"></td> | <td><img src="{diff_url}"></td> | ||||
| </tr>""" . format(tr_style=tr_style, | </tr>""" . format(tr_style=tr_style, | ||||
| name=name, | name=name, | ||||
| testname=testname, | testname=testname, | ||||
| kernel=kernel, | |||||
| status=status, | status=status, | ||||
| new_url=new_url, | new_url=new_url, | ||||
| ref_url=ref_url, | ref_url=ref_url, | ||||
| diff_url=diff_url) | diff_url=diff_url) | ||||
| if error: | if error: | ||||
| self.failed_tests += test_html | self.failed_tests += test_html | ||||
| else: | else: | ||||
| Show All 11 Lines | def _write_test_html(self, testname, filepath, error,kernel, allow_cb): | ||||
| name=name, | name=name, | ||||
| testname=testname, | testname=testname, | ||||
| status=status, | status=status, | ||||
| new_url=new_url, | new_url=new_url, | ||||
| ref_url=ref_url) | ref_url=ref_url) | ||||
| self.compare_tests += test_html | self.compare_tests += test_html | ||||
| def _diff_output(self, filepath, tmp_filepath): | def _diff_output(self, filepath, tmp_filepath,kernel, allow_cb): | ||||
| old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir) | old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir,kernel, allow_cb) | ||||
| # Create reference render directory. | # Create reference render directory. | ||||
| old_dirpath = os.path.dirname(old_img) | old_dirpath = os.path.dirname(old_img) | ||||
| os.makedirs(old_dirpath, exist_ok=True) | os.makedirs(old_dirpath, exist_ok=True) | ||||
| # Copy temporary to new image. | # Copy temporary to new image. | ||||
| if os.path.exists(new_img): | if os.path.exists(new_img): | ||||
| os.remove(new_img) | os.remove(new_img) | ||||
| Show All 40 Lines | def _diff_output(self, filepath, tmp_filepath,kernel, allow_cb): | ||||
| try: | try: | ||||
| subprocess.check_output(command) | subprocess.check_output(command) | ||||
| except subprocess.CalledProcessError as e: | except subprocess.CalledProcessError as e: | ||||
| if self.verbose: | if self.verbose: | ||||
| print_message(e.output.decode("utf-8")) | print_message(e.output.decode("utf-8")) | ||||
| return not failed | return not failed | ||||
| def _run_tests(self, filepaths, blender, arguments_cb, batch): | def _run_tests(self, filepaths, blender, arguments_cb, allow_cb, batch, kernel): | ||||
| # Run multiple tests in a single Blender process since startup can be | # Run multiple tests in a single Blender process since startup can be | ||||
| # a significant factor. In case of crashes, re-run the remaining tests. | # a significant factor. In case of crashes, re-run the remaining tests. | ||||
| verbose = os.environ.get("BLENDER_VERBOSE") is not None | verbose = os.environ.get("BLENDER_VERBOSE") is not None | ||||
| extra_verbose = verbose and os.environ.get("BLENDER_VERBOSE") > '1' | |||||
| remaining_filepaths = filepaths[:] | remaining_filepaths = filepaths[:] | ||||
| errors = [] | errors = [] | ||||
| while len(remaining_filepaths) > 0: | while len(remaining_filepaths) > 0: | ||||
| command = [blender] | command = [blender] | ||||
| output_filepaths = [] | output_filepaths = [] | ||||
| # Construct output filepaths and command to run | # Construct output filepaths and command to run | ||||
| for filepath in remaining_filepaths: | for filepath in remaining_filepaths: | ||||
| testname = test_get_name(filepath) | testname = test_get_name(filepath) + "_" + kernel | ||||
| print_message(testname, 'SUCCESS', 'RUN') | print_message(testname, 'SUCCESS', 'RUN') | ||||
| base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname) | base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname) | ||||
| output_filepath = base_output_filepath + '0001.png' | output_filepath = base_output_filepath + '0001.png' | ||||
| output_filepaths.append(output_filepath) | output_filepaths.append(output_filepath) | ||||
| if os.path.exists(output_filepath): | if os.path.exists(output_filepath): | ||||
| os.remove(output_filepath) | os.remove(output_filepath) | ||||
| command.extend(arguments_cb(filepath, base_output_filepath)) | command.extend(arguments_cb(filepath, base_output_filepath,kernel)) | ||||
| if extra_verbose: | |||||
| command.extend(['--debug-cycles']) | |||||
| # Only chain multiple commands for batch | # Only chain multiple commands for batch | ||||
| if not batch: | if not batch: | ||||
| break | break | ||||
| # Run process | # Run process | ||||
| crash = False | crash = False | ||||
| output = None | output = None | ||||
| try: | try: | ||||
| Show All 15 Lines | def _run_tests(self, filepaths, blender, arguments_cb, allow_cb, batch, kernel): | ||||
| if crash: | if crash: | ||||
| # In case of crash, stop after missing files and re-render remaining | # In case of crash, stop after missing files and re-render remaining | ||||
| if not os.path.exists(output_filepath): | if not os.path.exists(output_filepath): | ||||
| errors.append("CRASH") | errors.append("CRASH") | ||||
| print_message("Crash running Blender") | print_message("Crash running Blender") | ||||
| print_message(testname, 'FAILURE', 'FAILED') | print_message(testname, 'FAILURE', 'FAILED') | ||||
| break | break | ||||
| testname = test_get_name(filepath) | testname = test_get_name(filepath) + "_" + kernel | ||||
| if allow_cb(filepath, kernel) != "skip": | |||||
| if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0: | if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0: | ||||
| errors.append("NO OUTPUT") | errors.append("NO OUTPUT") | ||||
| print_message("No render result file found") | print_message("No render result file found") | ||||
| print_message(testname, 'FAILURE', 'FAILED') | print_message(testname, 'FAILURE', 'FAILED') | ||||
| elif not self._diff_output(filepath, output_filepath): | elif not self._diff_output(filepath, output_filepath,kernel, allow_cb): | ||||
| errors.append("VERIFY") | errors.append("VERIFY") | ||||
| print_message("Render result is different from reference image") | print_message("Render result is different from reference image") | ||||
| print_message(testname, 'FAILURE', 'FAILED') | print_message(testname, 'FAILURE', 'FAILED') | ||||
| else: | else: | ||||
| errors.append(None) | errors.append(None) | ||||
| print_message(testname, 'SUCCESS', 'OK') | print_message(testname, 'SUCCESS', 'OK') | ||||
| if os.path.exists(output_filepath): | if os.path.exists(output_filepath): | ||||
| os.remove(output_filepath) | os.remove(output_filepath) | ||||
| else: | |||||
| print_message(testname, 'SUCCESS', 'SKIP') | |||||
| return errors | return errors | ||||
| def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch): | def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, allow_cb, batch, kernel): | ||||
| passed_tests = [] | passed_tests = [] | ||||
| failed_tests = [] | failed_tests = [] | ||||
| all_files = list(blend_list(dirpath)) | all_files = list(blend_list(dirpath)) | ||||
| all_files = [x for x in all_files if allow_cb(x,kernel) != "skip"] | |||||
| all_files.sort() | all_files.sort() | ||||
| print_message("Running {} tests from 1 test case." . | print_message("Running {} tests from 1 test case." . | ||||
| format(len(all_files)), | format(len(all_files)), | ||||
| 'SUCCESS', "==========") | 'SUCCESS', "==========") | ||||
| time_start = time.time() | time_start = time.time() | ||||
| errors = self._run_tests(all_files, blender, arguments_cb, batch) | errors = self._run_tests(all_files, blender, arguments_cb, allow_cb, batch,kernel) | ||||
| for filepath, error in zip(all_files, errors): | for filepath, error in zip(all_files, errors): | ||||
| testname = test_get_name(filepath) | testname = test_get_name(filepath) | ||||
| if error: | if error: | ||||
| if error == "NO_ENGINE": | if error == "NO_ENGINE": | ||||
| return False | return False | ||||
| elif error == "NO_START": | elif error == "NO_START": | ||||
| return False | return False | ||||
| failed_tests.append(testname) | failed_tests.append(testname+"_"+kernel) | ||||
| else: | else: | ||||
| passed_tests.append(testname) | passed_tests.append(testname+"_"+kernel) | ||||
| self._write_test_html(dirname, filepath, error) | self._write_test_html(dirname, filepath, error,kernel, allow_cb) | ||||
| time_end = time.time() | time_end = time.time() | ||||
| elapsed_ms = int((time_end - time_start) * 1000) | elapsed_ms = int((time_end - time_start) * 1000) | ||||
| print_message("") | print_message("") | ||||
| print_message("{} tests from 1 test case ran. ({} ms total)" . | print_message("{} tests from 1 test case ran. ({} ms total)" . | ||||
| format(len(all_files), elapsed_ms), | format(len(all_files), elapsed_ms), | ||||
| 'SUCCESS', "==========") | 'SUCCESS', "==========") | ||||
| print_message("{} tests." . | print_message("{} tests." . | ||||
| format(len(passed_tests)), | format(len(passed_tests)), | ||||
| Show All 10 Lines | |||||