Differential D2593 Diff 8523 intern/cycles/device/device_cpu.cpp

Changeset View

Standalone View

intern/cycles/device/device_cpu.cpp

Show All 19 Lines
/* So ImathMath is included before our kernel_cpu_compat. */		/* So ImathMath is included before our kernel_cpu_compat. */
#ifdef WITH_OSL		#ifdef WITH_OSL
/* So no context pollution happens from indirectly included windows.h */		/* So no context pollution happens from indirectly included windows.h */
# include "util/util_windows.h"		# include "util/util_windows.h"
# include <OSL/oslexec.h>		# include <OSL/oslexec.h>
#endif		#endif

#include "device/device.h"		#include "device/device.h"
		#include "device/device_denoising.h"
#include "device/device_intern.h"		#include "device/device_intern.h"
#include "device/device_split_kernel.h"		#include "device/device_split_kernel.h"

#include "kernel/kernel.h"		#include "kernel/kernel.h"
#include "kernel/kernel_compat_cpu.h"		#include "kernel/kernel_compat_cpu.h"
#include "kernel/kernel_types.h"		#include "kernel/kernel_types.h"
#include "kernel/split/kernel_split_data.h"		#include "kernel/split/kernel_split_data.h"
#include "kernel/kernel_globals.h"		#include "kernel/kernel_globals.h"

		#include "filter/filter.h"

#include "kernel/osl/osl_shader.h"		#include "kernel/osl/osl_shader.h"
#include "kernel/osl/osl_globals.h"		#include "kernel/osl/osl_globals.h"

#include "render/buffers.h"		#include "render/buffers.h"

#include "util/util_debug.h"		#include "util/util_debug.h"
#include "util/util_foreach.h"		#include "util/util_foreach.h"
#include "util/util_function.h"		#include "util/util_function.h"
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines	#endif

DeviceRequestedFeatures requested_features;		DeviceRequestedFeatures requested_features;

KernelFunctions<void()(KernelGlobals , float , unsigned int , int, int, int, int, int)> path_trace_kernel;		KernelFunctions<void()(KernelGlobals , float , unsigned int , int, int, int, int, int)> path_trace_kernel;
KernelFunctions<void()(KernelGlobals , uchar4 , float , float, int, int, int, int)> convert_to_half_float_kernel;		KernelFunctions<void()(KernelGlobals , uchar4 , float , float, int, int, int, int)> convert_to_half_float_kernel;
KernelFunctions<void()(KernelGlobals , uchar4 , float , float, int, int, int, int)> convert_to_byte_kernel;		KernelFunctions<void()(KernelGlobals , uchar4 , float , float, int, int, int, int)> convert_to_byte_kernel;
KernelFunctions<void()(KernelGlobals , uint4 , float4 , float*, int, int, int, int, int)> shader_kernel;		KernelFunctions<void()(KernelGlobals , uint4 , float4 , float*, int, int, int, int, int)> shader_kernel;

		KernelFunctions<void()(int, TilesInfo, int, int, float, float, float, float, float, int, int, int, bool)> filter_divide_shadow_kernel;
		KernelFunctions<void()(int, TilesInfo, int, int, int, int, float, float, int*, int, int, bool)> filter_get_feature_kernel;
		KernelFunctions<void()(int, int, float, float, float, float, int, int)> filter_combine_halves_kernel;
		KernelFunctions<void()(int, int, int, float, int, int, int, int)> filter_divide_combined_kernel;

		KernelFunctions<void()(int, int, float, float, float, int*, int, int, float, float)> filter_nlm_calc_difference_kernel;
		KernelFunctions<void()(float, float, int, int, int)> filter_nlm_blur_kernel;
		KernelFunctions<void()(float, float, int, int, int)> filter_nlm_calc_weight_kernel;
		KernelFunctions<void()(int, int, float, float, float, float, int, int, int)> filter_nlm_update_output_kernel;
		KernelFunctions<void()(float, float, int, int)> filter_nlm_normalize_kernel;

		KernelFunctions<void()(int, float, int, int, int, float, int, int*, int, int, bool)> filter_construct_transform_kernel;
		KernelFunctions<void()(int, int, float, float, float, float, float, int, float, float3, int, int*, int, int, int, int)> filter_nlm_construct_gramian_kernel;
		KernelFunctions<void()(int, int, int, int, int, float, int, float, float3, int, int)> filter_finalize_kernel;

KernelFunctions<void()(KernelGlobals , ccl_constant KernelData, ccl_global void, int, ccl_global char*,		KernelFunctions<void()(KernelGlobals , ccl_constant KernelData, ccl_global void, int, ccl_global char*,
ccl_global uint, int, int, int, int, int, int, int, int, ccl_global int, int,		ccl_global uint, int, int, int, int, int, int, int, int, ccl_global int, int,
ccl_global char, ccl_global unsigned int, unsigned int, ccl_global float*)> data_init_kernel;		ccl_global char, ccl_global unsigned int, unsigned int, ccl_global float*)> data_init_kernel;
unordered_map<string, KernelFunctions<void()(KernelGlobals, KernelData*)> > split_kernels;		unordered_map<string, KernelFunctions<void()(KernelGlobals, KernelData*)> > split_kernels;

#define KERNEL_FUNCTIONS(name) \		#define KERNEL_FUNCTIONS(name) \
KERNEL_NAME_EVAL(cpu, name), \		KERNEL_NAME_EVAL(cpu, name), \
KERNEL_NAME_EVAL(cpu_sse2, name), \		KERNEL_NAME_EVAL(cpu_sse2, name), \
KERNEL_NAME_EVAL(cpu_sse3, name), \		KERNEL_NAME_EVAL(cpu_sse3, name), \
KERNEL_NAME_EVAL(cpu_sse41, name), \		KERNEL_NAME_EVAL(cpu_sse41, name), \
KERNEL_NAME_EVAL(cpu_avx, name), \		KERNEL_NAME_EVAL(cpu_avx, name), \
KERNEL_NAME_EVAL(cpu_avx2, name)		KERNEL_NAME_EVAL(cpu_avx2, name)

CPUDevice(DeviceInfo& info, Stats &stats, bool background)		CPUDevice(DeviceInfo& info, Stats &stats, bool background)
: Device(info, stats, background),		: Device(info, stats, background),
#define REGISTER_KERNEL(name) name ## _kernel(KERNEL_FUNCTIONS(name))		#define REGISTER_KERNEL(name) name ## _kernel(KERNEL_FUNCTIONS(name))
REGISTER_KERNEL(path_trace),		REGISTER_KERNEL(path_trace),
REGISTER_KERNEL(convert_to_half_float),		REGISTER_KERNEL(convert_to_half_float),
REGISTER_KERNEL(convert_to_byte),		REGISTER_KERNEL(convert_to_byte),
REGISTER_KERNEL(shader),		REGISTER_KERNEL(shader),
		REGISTER_KERNEL(filter_divide_shadow),
		REGISTER_KERNEL(filter_get_feature),
		REGISTER_KERNEL(filter_combine_halves),
		REGISTER_KERNEL(filter_divide_combined),
		REGISTER_KERNEL(filter_nlm_calc_difference),
		REGISTER_KERNEL(filter_nlm_blur),
		REGISTER_KERNEL(filter_nlm_calc_weight),
		REGISTER_KERNEL(filter_nlm_update_output),
		REGISTER_KERNEL(filter_nlm_normalize),
		REGISTER_KERNEL(filter_construct_transform),
		REGISTER_KERNEL(filter_nlm_construct_gramian),
		REGISTER_KERNEL(filter_finalize),
REGISTER_KERNEL(data_init)		REGISTER_KERNEL(data_init)
#undef REGISTER_KERNEL		#undef REGISTER_KERNEL
{		{

#ifdef WITH_OSL		#ifdef WITH_OSL
kernel_globals.osl = &osl_globals;		kernel_globals.osl = &osl_globals;
#endif		#endif
use_split_kernel = DebugFlags().cpu.split_kernel;		use_split_kernel = DebugFlags().cpu.split_kernel;
▲ Show 20 Lines • Show All 67 Lines • ▼ Show 20 Lines	#undef KERNEL_FUNCTIONS
}		}

void mem_free(device_memory& mem)		void mem_free(device_memory& mem)
{		{
if(mem.device_pointer) {		if(mem.device_pointer) {
if(!mem.data_pointer) {		if(!mem.data_pointer) {
free((void*)mem.device_pointer);		free((void*)mem.device_pointer);
}		}

mem.device_pointer = 0;		mem.device_pointer = 0;
stats.mem_free(mem.device_size);		stats.mem_free(mem.device_size);
mem.device_size = 0;		mem.device_size = 0;
}		}
}		}

		virtual device_ptr mem_get_offset_ptr(device_memory& mem, int offset, int /size/, MemoryType /type/)
		{
		return (device_ptr) (((char*) mem.device_pointer) + mem.memory_num_to_bytes(offset));
		}

void const_copy_to(const char name, void host, size_t size)		void const_copy_to(const char name, void host, size_t size)
{		{
kernel_const_copy(&kernel_globals, name, host, size);		kernel_const_copy(&kernel_globals, name, host, size);
}		}

void tex_alloc(const char *name,		void tex_alloc(const char *name,
device_memory& mem,		device_memory& mem,
InterpolationType interpolation,		InterpolationType interpolation,
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines	#endif
public:		public:
CPUDeviceTask(CPUDevice *device, DeviceTask& task)		CPUDeviceTask(CPUDevice *device, DeviceTask& task)
: DeviceTask(task)		: DeviceTask(task)
{		{
run = function_bind(&CPUDevice::thread_run, device, this);		run = function_bind(&CPUDevice::thread_run, device, this);
}		}
};		};

		bool denoising_set_tiles(device_ptr buffers, DenoisingTask task)
		{
		mem_alloc("Denoising Tile Info", task->tiles_mem, MEM_READ_ONLY);

		TilesInfo tiles = (TilesInfo) task->tiles_mem.data_pointer;
		for(int i = 0; i < 9; i++) {
		tiles->buffers[i] = buffers[i];
		}

		return true;
		}

		bool denoising_non_local_means(device_ptr image_ptr, device_ptr guide_ptr, device_ptr variance_ptr, device_ptr out_ptr,
		DenoisingTask *task)
		{
		int4 rect = task->rect;
		int r = task->nlm_state.r;
		int f = task->nlm_state.f;
		float a = task->nlm_state.a;
		float k_2 = task->nlm_state.k_2;

		int w = align_up(rect.z-rect.x, 4);
		int h = rect.w-rect.y;

		float blurDifference = (float) task->nlm_state.temporary_1_ptr;
		float difference = (float) task->nlm_state.temporary_2_ptr;
		float weightAccum = (float) task->nlm_state.temporary_3_ptr;

		memset(weightAccum, 0, sizeof(float)wh);
		memset((float) out_ptr, 0, sizeof(float)w*h);

		for(int i = 0; i < (2r+1)(2*r+1); i++) {
		int dy = i / (2*r+1) - r;
		int dx = i % (2*r+1) - r;

		int local_rect[4] = {max(0, -dx), max(0, -dy), rect.z-rect.x - max(0, dx), rect.w-rect.y - max(0, dy)};
		filter_nlm_calc_difference_kernel()(dx, dy,
		(float*) guide_ptr,
		(float*) variance_ptr,
		difference,
		local_rect,
		w, 0,
		a, k_2);

		filter_nlm_blur_kernel() (difference, blurDifference, local_rect, w, f);
		filter_nlm_calc_weight_kernel()(blurDifference, difference, local_rect, w, f);
		filter_nlm_blur_kernel() (difference, blurDifference, local_rect, w, f);

		filter_nlm_update_output_kernel()(dx, dy,
		blurDifference,
		(float*) image_ptr,
		(float*) out_ptr,
		weightAccum,
		local_rect,
		w, f);
		}

		int local_rect[4] = {0, 0, rect.z-rect.x, rect.w-rect.y};
		filter_nlm_normalize_kernel()((float*) out_ptr, weightAccum, local_rect, w);

		return true;
		}

		bool denoising_construct_transform(DenoisingTask *task)
		{
		for(int y = 0; y < task->filter_area.w; y++) {
		for(int x = 0; x < task->filter_area.z; x++) {
		filter_construct_transform_kernel()(task->render_buffer.samples,
		(float*) task->buffer.mem.device_pointer,
		x + task->filter_area.x,
		y + task->filter_area.y,
		y*task->filter_area.z + x,
		(float*) task->storage.transform.device_pointer,
		(int*) task->storage.rank.device_pointer,
		&task->rect.x,
		task->buffer.pass_stride,
		task->radius,
		task->relative_pca);
		}
		}
		return true;
		}

		bool denoising_reconstruct(device_ptr color_ptr,
		device_ptr color_variance_ptr,
		device_ptr guide_ptr,
		device_ptr guide_variance_ptr,
		device_ptr output_ptr,
		DenoisingTask *task)
		{
		mem_zero(task->storage.XtWX);
		mem_zero(task->storage.XtWY);

		float difference = (float) task->reconstruction_state.temporary_1_ptr;
		float blurDifference = (float) task->reconstruction_state.temporary_2_ptr;

		int r = task->radius;
		for(int i = 0; i < (2r+1)(2*r+1); i++) {
		int dy = i / (2*r+1) - r;
		int dx = i % (2*r+1) - r;

		int local_rect[4] = {max(0, -dx), max(0, -dy),
		task->reconstruction_state.source_w - max(0, dx),
		task->reconstruction_state.source_h - max(0, dy)};
		filter_nlm_calc_difference_kernel()(dx, dy,
		(float*) guide_ptr,
		(float*) guide_variance_ptr,
		difference,
		local_rect,
		task->buffer.w,
		task->buffer.pass_stride,
		1.0f,
		task->nlm_k_2);
		filter_nlm_blur_kernel()(difference, blurDifference, local_rect, task->buffer.w, 4);
		filter_nlm_calc_weight_kernel()(blurDifference, difference, local_rect, task->buffer.w, 4);
		filter_nlm_blur_kernel()(difference, blurDifference, local_rect, task->buffer.w, 4);
		filter_nlm_construct_gramian_kernel()(dx, dy,
		blurDifference,
		(float*) task->buffer.mem.device_pointer,
		(float*) color_ptr,
		(float*) color_variance_ptr,
		(float*) task->storage.transform.device_pointer,
		(int*) task->storage.rank.device_pointer,
		(float*) task->storage.XtWX.device_pointer,
		(float3*) task->storage.XtWY.device_pointer,
		local_rect,
		&task->reconstruction_state.filter_rect.x,
		task->buffer.w,
		task->buffer.h,
		4,
		task->buffer.pass_stride);
		}
		for(int y = 0; y < task->filter_area.w; y++) {
		for(int x = 0; x < task->filter_area.z; x++) {
		filter_finalize_kernel()(x,
		y,
		y*task->filter_area.z + x,
		task->buffer.w,
		task->buffer.h,
		(float*) output_ptr,
		(int*) task->storage.rank.device_pointer,
		(float*) task->storage.XtWX.device_pointer,
		(float3*) task->storage.XtWY.device_pointer,
		&task->reconstruction_state.buffer_params.x,
		task->render_buffer.samples);
		}
		}
		return true;
		}

		bool denoising_combine_halves(device_ptr a_ptr, device_ptr b_ptr,
		device_ptr mean_ptr, device_ptr variance_ptr,
		int r, int4 rect, DenoisingTask *task)
		{
		(void) task;
		for(int y = rect.y; y < rect.w; y++) {
		for(int x = rect.x; x < rect.z; x++) {
		filter_combine_halves_kernel()(x, y,
		(float*) mean_ptr,
		(float*) variance_ptr,
		(float*) a_ptr,
		(float*) b_ptr,
		&rect.x,
		r);
		}
		}
		return true;
		}

		bool denoising_divide_shadow(device_ptr a_ptr, device_ptr b_ptr,
		device_ptr sample_variance_ptr, device_ptr sv_variance_ptr,
		device_ptr buffer_variance_ptr, DenoisingTask *task)
		{
		for(int y = task->rect.y; y < task->rect.w; y++) {
		for(int x = task->rect.x; x < task->rect.z; x++) {
		filter_divide_shadow_kernel()(task->render_buffer.samples,
		task->tiles,
		x, y,
		(float*) a_ptr,
		(float*) b_ptr,
		(float*) sample_variance_ptr,
		(float*) sv_variance_ptr,
		(float*) buffer_variance_ptr,
		&task->rect.x,
		task->render_buffer.pass_stride,
		task->render_buffer.denoising_data_offset,
		task->use_split_variance);
		}
		}
		return true;
		}

		bool denoising_get_feature(int mean_offset,
		int variance_offset,
		device_ptr mean_ptr,
		device_ptr variance_ptr,
		DenoisingTask *task)
		{
		for(int y = task->rect.y; y < task->rect.w; y++) {
		for(int x = task->rect.x; x < task->rect.z; x++) {
		filter_get_feature_kernel()(task->render_buffer.samples,
		task->tiles,
		mean_offset,
		variance_offset,
		x, y,
		(float*) mean_ptr,
		(float*) variance_ptr,
		&task->rect.x,
		task->render_buffer.pass_stride,
		task->render_buffer.denoising_data_offset,
		task->use_split_variance);
		}
		}
		return true;
		}

void path_trace(DeviceTask &task, RenderTile &tile, KernelGlobals *kg)		void path_trace(DeviceTask &task, RenderTile &tile, KernelGlobals *kg)
{		{
float render_buffer = (float)tile.buffer;		float render_buffer = (float)tile.buffer;
uint rng_state = (uint)tile.rng_state;		uint rng_state = (uint)tile.rng_state;
int start_sample = tile.start_sample;		int start_sample = tile.start_sample;
int end_sample = tile.start_sample + tile.num_samples;		int end_sample = tile.start_sample + tile.num_samples;

for(int sample = start_sample; sample < end_sample; sample++) {		for(int sample = start_sample; sample < end_sample; sample++) {
Show All 10 Lines	for(int sample = start_sample; sample < end_sample; sample++) {
}		}

tile.sample = sample + 1;		tile.sample = sample + 1;

task.update_progress(&tile, tile.w*tile.h);		task.update_progress(&tile, tile.w*tile.h);
}		}
}		}

		void denoise(DeviceTask &task, RenderTile &tile)
		{
		tile.sample = tile.start_sample + tile.num_samples;

		DenoisingTask denoising(this);

		denoising.functions.construct_transform = function_bind(&CPUDevice::denoising_construct_transform, this, &denoising);
		denoising.functions.reconstruct = function_bind(&CPUDevice::denoising_reconstruct, this, _1, _2, _3, _4, _5, &denoising);
		denoising.functions.divide_shadow = function_bind(&CPUDevice::denoising_divide_shadow, this, _1, _2, _3, _4, _5, &denoising);
		denoising.functions.non_local_means = function_bind(&CPUDevice::denoising_non_local_means, this, _1, _2, _3, _4, &denoising);
		denoising.functions.combine_halves = function_bind(&CPUDevice::denoising_combine_halves, this, _1, _2, _3, _4, _5, _6, &denoising);
		denoising.functions.get_feature = function_bind(&CPUDevice::denoising_get_feature, this, _1, _2, _3, _4, &denoising);
		denoising.functions.set_tiles = function_bind(&CPUDevice::denoising_set_tiles, this, _1, &denoising);

		denoising.filter_area = make_int4(tile.x, tile.y, tile.w, tile.h);
		denoising.render_buffer.samples = tile.sample;
		denoising.use_split_variance = use_split_kernel;

		RenderTile rtiles[9];
		rtiles[4] = tile;
		task.get_neighbor_tiles(rtiles, this);
		denoising.tiles_from_rendertiles(rtiles);

		denoising.init_from_devicetask(task);

		denoising.run_denoising();

		task.release_neighbor_tiles(rtiles, this);

		task.update_progress(&tile, tile.w*tile.h);
		}

void thread_render(DeviceTask& task)		void thread_render(DeviceTask& task)
{		{
if(task_pool.canceled()) {		if(task_pool.canceled()) {
if(task.need_finish_queue == false)		if(task.need_finish_queue == false)
return;		return;
}		}

/* allocate buffer for kernel globals */		/* allocate buffer for kernel globals */
Show All 13 Lines	if(use_split_kernel) {
mem_free(kgbuffer);		mem_free(kgbuffer);

return;		return;
}		}
}		}

RenderTile tile;		RenderTile tile;
while(task.acquire_tile(this, tile)) {		while(task.acquire_tile(this, tile)) {
		if(tile.task == RenderTile::PATH_TRACE) {
if(use_split_kernel) {		if(use_split_kernel) {
device_memory data;		device_memory data;
split_kernel->path_trace(&task, tile, kgbuffer, data);		split_kernel->path_trace(&task, tile, kgbuffer, data);
}		}
else {		else {
path_trace(task, tile, kg);		path_trace(task, tile, kg);
}		}
		}
		else if(tile.task == RenderTile::DENOISE) {
		denoise(task, tile);
		}

task.release_tile(tile);		task.release_tile(tile);

if(task_pool.canceled()) {		if(task_pool.canceled()) {
if(task.need_finish_queue == false)		if(task.need_finish_queue == false)
break;		break;
}		}
}		}
▲ Show 20 Lines • Show All 273 Lines • Show Last 20 Lines