Differential D13423 Diff 45772 intern/cycles/device/metal/kernel.mm

Changeset View

Standalone View

intern/cycles/device/metal/kernel.mm

This file was added.

				/*
				* Copyright 2021 Blender Foundation
				*
				* Licensed under the Apache License, Version 2.0 (the "License");
				* you may not use this file except in compliance with the License.
				* You may obtain a copy of the License at
				*
				* http://www.apache.org/licenses/LICENSE-2.0
				*
				* Unless required by applicable law or agreed to in writing, software
				* distributed under the License is distributed on an "AS IS" BASIS,
				* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
				* See the License for the specific language governing permissions and
				* limitations under the License.
				*/

				#ifdef WITH_METAL

				# include "device/metal/kernel.h"
				# include "device/metal/device_impl.h"
				# include "util/md5.h"
				# include "util/path.h"
				# include "util/tbb.h"
				# include "util/time.h"

				CCL_NAMESPACE_BEGIN

				/* limit to 2 MTLCompiler instances */
				int max_mtlcompiler_threads = 2;

				const char *kernel_type_as_string(int kernel_type)
				{
				switch (kernel_type) {
				case PSO_GENERIC:
				return "PSO_GENERIC";
				case PSO_SPECIALISED:
				return "PSO_SPECIALISED";
				default:
				assert(0);
				}
				return "";
				}

				MetalDeviceKernel::~MetalDeviceKernel()
				{
				for (int i = 0; i < PSO_NUM; i++) {
				pso[i].release();
				}
				}

				bool MetalDeviceKernel::load(MetalDevice *device,
				MetalKernelLoadDesc const &desc_in,
				MD5Hash const &md5)
				{
				__block MetalKernelLoadDesc const desc(desc_in);
				if (desc.kernel_index == DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL) {
				/* skip megakernel */
				return true;
				}

				bool use_binary_archive = true;
				if (getenv("CYCLES_METAL_DISABLE_BINARY_ARCHIVES")) {
				use_binary_archive = false;
				}

				id<MTLBinaryArchive> archive = nil;
				string metalbin_path;
				if (use_binary_archive) {
				NSProcessInfo *processInfo = [NSProcessInfo processInfo];
				string osVersion = [[processInfo operatingSystemVersionString] UTF8String];
				MD5Hash local_md5(md5);
				local_md5.append(osVersion);
				string metalbin_name = string(desc.function_name) + "." + local_md5.get_hex() +
				to_string(desc.pso_index) + ".bin";
				metalbin_path = path_cache_get(path_join("kernels", metalbin_name));
				path_create_directories(metalbin_path);

				if (path_exists(metalbin_path) && use_binary_archive) {
				if (@available(macOS 11.0, *)) {
				MTLBinaryArchiveDescriptor *archiveDesc = [[MTLBinaryArchiveDescriptor alloc] init];
				archiveDesc.url = [NSURL fileURLWithPath:@(metalbin_path.c_str())];
				archive = [device->mtlDevice newBinaryArchiveWithDescriptor:archiveDesc error:nil];
				[archiveDesc release];
				}
				}
				}

				NSString *entryPoint = [@(desc.function_name) copy];

				NSError *error = NULL;
				if (@available(macOS 11.0, *)) {
				MTLFunctionDescriptor *func_desc = [MTLIntersectionFunctionDescriptor functionDescriptor];
				func_desc.name = entryPoint;
				if (desc.constant_values) {
				func_desc.constantValues = desc.constant_values;
				}
				pso[desc.pso_index].function = [device->mtlLibrary[desc.pso_index]
				newFunctionWithDescriptor:func_desc
				error:&error];
				}
				[entryPoint release];

				if (pso[desc.pso_index].function == nil) {
				NSString *err = [error localizedDescription];
				string errors = [err UTF8String];

				device->set_error(
				string_printf("Error getting function \"%s\": %s", desc.function_name, errors.c_str()));
				return false;
				}

				pso[desc.pso_index].function.label = [@(desc.function_name) copy];

				__block MTLComputePipelineDescriptor *computePipelineStateDescriptor =
				[[MTLComputePipelineDescriptor alloc] init];

				computePipelineStateDescriptor.buffers[0].mutability = MTLMutabilityImmutable;
				computePipelineStateDescriptor.buffers[1].mutability = MTLMutabilityImmutable;
				computePipelineStateDescriptor.buffers[2].mutability = MTLMutabilityImmutable;

				computePipelineStateDescriptor.maxTotalThreadsPerThreadgroup = desc.threads_per_threadgroup;
				computePipelineStateDescriptor.threadGroupSizeIsMultipleOfThreadExecutionWidth = true;

				computePipelineStateDescriptor.computeFunction = pso[desc.pso_index].function;
				if (@available(macOS 11.0, *)) {
				/* Attach the additional functions to an MTLLinkedFunctions object */
				if (desc.linked_functions) {
				computePipelineStateDescriptor.linkedFunctions = [[MTLLinkedFunctions alloc] init];
				computePipelineStateDescriptor.linkedFunctions.functions = desc.linked_functions;
				}

				computePipelineStateDescriptor.maxCallStackDepth = 1;
				}

				/* Create a new Compute pipeline state object */
				MTLPipelineOption pipelineOptions = MTLPipelineOptionNone;

				bool creating_new_archive = false;
				if (@available(macOS 11.0, *)) {
				if (use_binary_archive) {
				if (!archive) {
				MTLBinaryArchiveDescriptor *archiveDesc = [[MTLBinaryArchiveDescriptor alloc] init];
				archiveDesc.url = nil;
				archive = [device->mtlDevice newBinaryArchiveWithDescriptor:archiveDesc error:nil];
				creating_new_archive = true;

				double starttime = time_dt();

				if (![archive addComputePipelineFunctionsWithDescriptor:computePipelineStateDescriptor
				error:&error]) {
				NSString *errStr = [error localizedDescription];
				metal_printf("Failed to add PSO to archive:\n%s\n",
				errStr ? [errStr UTF8String] : "nil");
				}
				else {
				double duration = time_dt() - starttime;
				metal_printf("%2d \| %-55s \| %7.2fs\n",
				desc.kernel_index,
				device_kernel_as_string((DeviceKernel)desc.kernel_index),
				duration);

				if (desc.pso_index == PSO_GENERIC) {
				this->load_duration = duration;
				}
				}
				}
				computePipelineStateDescriptor.binaryArchives = [NSArray arrayWithObjects:archive, nil];
				pipelineOptions = MTLPipelineOptionFailOnBinaryArchiveMiss;
				}
				}

				double starttime = time_dt();

				MTLNewComputePipelineStateWithReflectionCompletionHandler completionHandler = ^(
				id<MTLComputePipelineState> computePipelineState,
				MTLComputePipelineReflection *reflection,
				NSError *error) {
				bool recreate_archive = false;
				if (computePipelineState == nil && archive && !creating_new_archive) {

				assert(0);

				NSString *errStr = [error localizedDescription];
				metal_printf(
				"Failed to create compute pipeline state \"%s\" from archive - attempting recreation... "
				"(error: %s)\n",
				device_kernel_as_string((DeviceKernel)desc.kernel_index),
				errStr ? [errStr UTF8String] : "nil");
				computePipelineState = [device->mtlDevice
				newComputePipelineStateWithDescriptor:computePipelineStateDescriptor
				options:MTLPipelineOptionNone
				reflection:nullptr
				error:&error];
				recreate_archive = true;
				}

				double duration = time_dt() - starttime;

				if (computePipelineState == nil) {
				NSString *errStr = [error localizedDescription];
				device->set_error(string_printf("Failed to create compute pipeline state \"%s\", error: \n",
				device_kernel_as_string((DeviceKernel)desc.kernel_index)) +
				(errStr ? [errStr UTF8String] : "nil"));
				metal_printf("%2d \| %-55s \| %7.2fs \| FAILED!\n",
				desc.kernel_index,
				device_kernel_as_string((DeviceKernel)desc.kernel_index),
				duration);
				return;
				}

				pso[desc.pso_index].pipeline = computePipelineState;
				num_threads_per_block = round_down(computePipelineState.maxTotalThreadsPerThreadgroup,
				computePipelineState.threadExecutionWidth);
				num_threads_per_block = std::max(num_threads_per_block,
				(int)computePipelineState.threadExecutionWidth);

				if (!use_binary_archive) {
				metal_printf("%2d \| %-55s \| %7.2fs\n",
				desc.kernel_index,
				device_kernel_as_string((DeviceKernel)desc.kernel_index),
				duration);

				if (desc.pso_index == PSO_GENERIC) {
				this->load_duration = duration;
				}
				}

				if (@available(macOS 11.0, *)) {
				if (creating_new_archive \|\| recreate_archive) {
				if (![archive serializeToURL:[NSURL fileURLWithPath:@(metalbin_path.c_str())]
				error:&error]) {
				metal_printf("Failed to save binary archive, error:\n%s\n",
				[[error localizedDescription] UTF8String]);
				}
				}
				}

				[computePipelineStateDescriptor release];
				computePipelineStateDescriptor = nil;

				if (device->use_metalrt && desc.linked_functions) {
				for (int table = 0; table < METALRT_TABLE_NUM; table++) {
				if (@available(macOS 11.0, *)) {
				MTLIntersectionFunctionTableDescriptor *ift_desc =
				[[MTLIntersectionFunctionTableDescriptor alloc] init];
				ift_desc.functionCount = desc.intersector_functions[table].count;

				pso[desc.pso_index].intersection_func_table[table] = [pso[desc.pso_index].pipeline
				newIntersectionFunctionTableWithDescriptor:ift_desc];

				/* Finally write the function handles into this pipeline's table */
				for (int i = 0; i < 2; i++) {
				id<MTLFunctionHandle> handle = [pso[desc.pso_index].pipeline
				functionHandleWithFunction:desc.intersector_functions[table][i]];
				[pso[desc.pso_index].intersection_func_table[table] setFunction:handle atIndex:i];
				}
				}
				}
				}

				mark_loaded(desc.pso_index);
				};

				if (desc.pso_index == PSO_SPECIALISED) {
				/* Asynchronous load */
				dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
				NSError *error;
				id<MTLComputePipelineState> pipeline = [device->mtlDevice
				newComputePipelineStateWithDescriptor:computePipelineStateDescriptor
				options:pipelineOptions
				reflection:nullptr
				error:&error];
				completionHandler(pipeline, nullptr, error);
				});
				}
				else {
				/* Block on load to ensure we continue with a valid kernel function */
				id<MTLComputePipelineState> pipeline = [device->mtlDevice
				newComputePipelineStateWithDescriptor:computePipelineStateDescriptor
				options:pipelineOptions
				reflection:nullptr
				error:&error];
				completionHandler(pipeline, nullptr, error);
				}

				return true;
				}

				const MetalKernelPipeline &MetalDeviceKernel::get_pso() const
				{
				if (pso[PSO_SPECIALISED].loaded) {
				return pso[PSO_SPECIALISED];
				}

				assert(pso[PSO_GENERIC].loaded);
				return pso[PSO_GENERIC];
				}

				bool MetalDeviceKernels::load(MetalDevice *device, int kernel_type)
				{
				bool any_error = false;

				MD5Hash md5;

				/* Build the function constant table */
				MTLFunctionConstantValues *constant_values = nullptr;
				if (kernel_type == PSO_SPECIALISED) {
				constant_values = [MTLFunctionConstantValues new];

				# define KERNEL_FILM(_type, name) \
				[constant_values setConstantValue:&data.film.name \
				type:get_MTLDataType_##_type() \
				atIndex:KernelData_film_##name]; \
				md5.append((uint8_t *)&data.film.name, sizeof(data.film.name));

				# define KERNEL_BACKGROUND(_type, name) \
				[constant_values setConstantValue:&data.background.name \
				type:get_MTLDataType_##_type() \
				atIndex:KernelData_background_##name]; \
				md5.append((uint8_t *)&data.background.name, sizeof(data.background.name));

				# define KERNEL_INTEGRATOR(_type, name) \
				[constant_values setConstantValue:&data.integrator.name \
				type:get_MTLDataType_##_type() \
				atIndex:KernelData_integrator_##name]; \
				md5.append((uint8_t *)&data.integrator.name, sizeof(data.integrator.name));

				# define KERNEL_BVH(_type, name) \
				[constant_values setConstantValue:&data.bvh.name \
				type:get_MTLDataType_##_type() \
				atIndex:KernelData_bvh_##name]; \
				md5.append((uint8_t *)&data.bvh.name, sizeof(data.bvh.name));

				/* METAL_WIP: populate constant_values based on KernelData */
				assert(0);
				/*
				const KernelData &data = device->launch_params.data;
				# include "kernel/types/background.h"
				# include "kernel/types/bvh.h"
				# include "kernel/types/film.h"
				# include "kernel/types/integrator.h"
				*/
				}

				if (device->use_metalrt) {
				if (@available(macOS 11.0, *)) {
				/* create the id<MTLFunction> for each intersection function */
				const char *function_names[] = {
				"__anyhit__cycles_metalrt_visibility_test_tri",
				"__anyhit__cycles_metalrt_visibility_test_box",
				"__anyhit__cycles_metalrt_shadow_all_hit_tri",
				"__anyhit__cycles_metalrt_shadow_all_hit_box",
				"__anyhit__cycles_metalrt_local_hit_tri",
				"__anyhit__cycles_metalrt_local_hit_box",
				"__intersection__curve_ribbon",
				"__intersection__curve_ribbon_shadow",
				"__intersection__curve_all",
				"__intersection__curve_all_shadow",
				};
				assert(sizeof(function_names) / sizeof(function_names[0]) == METALRT_FUNC_NUM);

				MTLFunctionDescriptor *desc = [MTLIntersectionFunctionDescriptor functionDescriptor];
				if (kernel_type == PSO_SPECIALISED) {
				desc.constantValues = constant_values;
				}
				for (int i = 0; i < METALRT_FUNC_NUM; i++) {
				const char *function_name = function_names[i];
				desc.name = [@(function_name) copy];

				NSError *error = NULL;
				rt_intersection_funcs[kernel_type][i] = [device->mtlLibrary[kernel_type]
				newFunctionWithDescriptor:desc
				error:&error];

				if (rt_intersection_funcs[kernel_type][i] == nil) {
				NSString *err = [error localizedDescription];
				string errors = [err UTF8String];

				device->set_error(string_printf(
				"Error getting intersection function \"%s\": %s", function_name, errors.c_str()));
				any_error = true;
				break;
				}

				rt_intersection_funcs[kernel_type][i].label = [@(function_name) copy];
				}
				}
				}
				md5.append(device->source_used_for_compile[kernel_type]);

				string hash = md5.get_hex();
				if (loaded_md5[kernel_type] == hash) {
				return true;
				}

				if (!any_error) {
				NSArray *table_functions[METALRT_TABLE_NUM] = {nil};
				NSArray *function_list = nil;

				if (device->use_metalrt) {
				id<MTLFunction> box_intersect_default = nil;
				id<MTLFunction> box_intersect_shadow = nil;
				if (device->kernel_features & KERNEL_FEATURE_HAIR) {
				/* Add curve intersection programs. */
				if (device->kernel_features & KERNEL_FEATURE_HAIR_THICK) {
				/* Slower programs for thick hair since that also slows down ribbons.
				* Ideally this should not be needed. */
				box_intersect_default = rt_intersection_funcs[kernel_type][METALRT_FUNC_CURVE_ALL];
				box_intersect_shadow = rt_intersection_funcs[kernel_type][METALRT_FUNC_CURVE_ALL_SHADOW];
				}
				else {
				box_intersect_default = rt_intersection_funcs[kernel_type][METALRT_FUNC_CURVE_RIBBON];
				box_intersect_shadow =
				rt_intersection_funcs[kernel_type][METALRT_FUNC_CURVE_RIBBON_SHADOW];
				}
				}
				table_functions[METALRT_TABLE_DEFAULT] = [NSArray
				arrayWithObjects:rt_intersection_funcs[kernel_type][METALRT_FUNC_DEFAULT_TRI],
				box_intersect_default ?
				box_intersect_default :
				rt_intersection_funcs[kernel_type][METALRT_FUNC_DEFAULT_BOX],
				nil];
				table_functions[METALRT_TABLE_SHADOW] = [NSArray
				arrayWithObjects:rt_intersection_funcs[kernel_type][METALRT_FUNC_SHADOW_TRI],
				box_intersect_shadow ?
				box_intersect_shadow :
				rt_intersection_funcs[kernel_type][METALRT_FUNC_SHADOW_BOX],
				nil];
				table_functions[METALRT_TABLE_LOCAL] = [NSArray
				arrayWithObjects:rt_intersection_funcs[kernel_type][METALRT_FUNC_LOCAL_TRI],
				rt_intersection_funcs[kernel_type][METALRT_FUNC_LOCAL_BOX],
				nil];

				NSMutableSet *unique_functions = [NSMutableSet
				setWithArray:table_functions[METALRT_TABLE_DEFAULT]];
				[unique_functions addObjectsFromArray:table_functions[METALRT_TABLE_SHADOW]];
				[unique_functions addObjectsFromArray:table_functions[METALRT_TABLE_LOCAL]];

				function_list = [[NSArray arrayWithArray:[unique_functions allObjects]]
				sortedArrayUsingComparator:^NSComparisonResult(id<MTLFunction> f1, id<MTLFunction> f2) {
				return [f1.label compare:f2.label];
				}];

				unique_functions = nil;
				}

				metal_printf("Starting %s \"cycles_metal_...\" pipeline builds\n",
				kernel_type_as_string(kernel_type));

				tbb::task_arena local_arena(max_mtlcompiler_threads);
				local_arena.execute([&]() {
				tbb::parallel_for(int(0), int(DEVICE_KERNEL_NUM), [&](int i) {
				/* skip megakernel */
				if (i == DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL) {
				return;
				}

				/* Only specialise kernels where it can make an impact */
				if (kernel_type == PSO_SPECIALISED) {
				if (i < DEVICE_KERNEL_INTEGRATOR_INTERSECT_CLOSEST \|\|
				i > DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL) {
				return;
				}
				}

				MetalDeviceKernel &kernel = kernels_[i];

				const std::string function_name = std::string("cycles_metal_") +
				device_kernel_as_string((DeviceKernel)i);
				int threads_per_threadgroup = device->max_threads_per_threadgroup;
				if (i > DEVICE_KERNEL_INTEGRATOR_MEGAKERNEL && i < DEVICE_KERNEL_INTEGRATOR_RESET) {
				/* Always use 512 for the sorting kernels */
				threads_per_threadgroup = 512;
				}

				NSArray *kernel_function_list = nil;

				if (i == DEVICE_KERNEL_INTEGRATOR_INTERSECT_CLOSEST \|\|
				i == DEVICE_KERNEL_INTEGRATOR_INTERSECT_SHADOW \|\|
				i == DEVICE_KERNEL_INTEGRATOR_INTERSECT_SUBSURFACE \|\|
				i == DEVICE_KERNEL_INTEGRATOR_INTERSECT_VOLUME_STACK \|\|
				i == DEVICE_KERNEL_INTEGRATOR_SHADE_SURFACE_RAYTRACE) {
				kernel_function_list = function_list;
				}

				MetalKernelLoadDesc desc;
				desc.pso_index = kernel_type;
				desc.kernel_index = i;
				desc.linked_functions = kernel_function_list;
				desc.intersector_functions.defaults = table_functions[METALRT_TABLE_DEFAULT];
				desc.intersector_functions.shadow = table_functions[METALRT_TABLE_SHADOW];
				desc.intersector_functions.local = table_functions[METALRT_TABLE_LOCAL];
				desc.constant_values = constant_values;
				desc.threads_per_threadgroup = threads_per_threadgroup;
				desc.function_name = function_name.c_str();

				bool success = kernel.load(device, desc, md5);

				any_error \|= !success;
				});
				});
				}

				bool loaded = !any_error;
				if (loaded) {
				loaded_md5[kernel_type] = hash;
				}
				return loaded;
				}

				const MetalDeviceKernel &MetalDeviceKernels::get(DeviceKernel kernel) const
				{
				return kernels_[(int)kernel];
				}

				bool MetalDeviceKernels::available(DeviceKernel kernel) const
				{
				return kernels_[(int)kernel].get_pso().function != nil;
				}

				CCL_NAMESPACE_END

				#endif /* WITH_METAL*/