Differential D7475 Diff 23907 source/blender/blenlib/intern/task_pool_legacy.cc

Changeset View

Standalone View

source/blender/blenlib/intern/task_pool_legacy.cc

This file was copied from source/blender/blenlib/intern/task_pool.cc.

Show All 39 Lines
/* Types */		/* Types */

/* Number of per-thread pre-allocated tasks.		/* Number of per-thread pre-allocated tasks.
*		*
* For more details see description of TaskMemPool.		* For more details see description of TaskMemPool.
*/		*/
#define MEMPOOL_SIZE 256		#define MEMPOOL_SIZE 256

/* Number of tasks which are pushed directly to local thread queue.
*
* This allows thread to fetch next task without locking the whole queue.
*/
#define LOCAL_QUEUE_SIZE 1

/* Number of tasks which are allowed to be scheduled in a delayed manner.
*
* This allows to use less locks per graph node children schedule. More details
* could be found at TaskThreadLocalStorage::do_delayed_push.
*/
#define DELAYED_QUEUE_SIZE 4096

#ifndef NDEBUG		#ifndef NDEBUG
# define ASSERT_THREAD_ID(scheduler, thread_id) \		# define ASSERT_THREAD_ID(scheduler, thread_id) \
do { \		do { \
if (!BLI_thread_is_main()) { \		if (!BLI_thread_is_main()) { \
TaskThread thread = (TaskThread )pthread_getspecific(scheduler->tls_id_key); \		TaskThread thread = (TaskThread )pthread_getspecific(scheduler->tls_id_key); \
if (thread == NULL) { \		if (thread == NULL) { \
BLI_assert(thread_id == 0); \		BLI_assert(thread_id == 0); \
} \		} \
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines	typedef struct TaskMemPoolStats {
int num_alloc;		int num_alloc;
/* Number of avoided allocations (pointer was re-used from the pool). */		/* Number of avoided allocations (pointer was re-used from the pool). */
int num_reuse;		int num_reuse;
/* Number of discarded memory due to pool saturation, */		/* Number of discarded memory due to pool saturation, */
int num_discard;		int num_discard;
} TaskMemPoolStats;		} TaskMemPoolStats;
#endif		#endif

typedef struct TaskThreadLocalStorage {
/* Memory pool for faster task allocation.
* The idea is to re-use memory of finished/discarded tasks by this thread.
*/
TaskMemPool task_mempool;

/* Local queue keeps thread alive by keeping small amount of tasks ready
* to be picked up without causing global thread locks for synchronization.
*/
int num_local_queue;
Task *local_queue[LOCAL_QUEUE_SIZE];

/* Thread can be marked for delayed tasks push. This is helpful when it's
* know that lots of subsequent task pushed will happen from the same thread
* without "interrupting" for task execution.
*
* We try to accumulate as much tasks as possible in a local queue without
* any locks first, and then we push all of them into a scheduler's queue
* from within a single mutex lock.
*/
bool do_delayed_push;
int num_delayed_queue;
Task *delayed_queue[DELAYED_QUEUE_SIZE];
} TaskThreadLocalStorage;

struct TaskPool {		struct TaskPool {
TaskScheduler *scheduler;		TaskScheduler *scheduler;

volatile size_t num;		volatile size_t num;
ThreadMutex num_mutex;		ThreadMutex num_mutex;
ThreadCondition num_cond;		ThreadCondition num_cond;

void *userdata;		void *userdata;
Show All 14 Lines	struct TaskPool {
* single-threaded situation.		* single-threaded situation.
*/		*/
bool run_in_background;		bool run_in_background;

/* This is a task scheduler's ID of a thread at which pool was constructed.		/* This is a task scheduler's ID of a thread at which pool was constructed.
* It will be used to access task TLS.		* It will be used to access task TLS.
*/		*/
int thread_id;		int thread_id;

/* For the pools which are created from non-main thread which is not a
* scheduler worker thread we can't re-use any of scheduler's threads TLS
* and have to use our own one.
*/
bool use_local_tls;
TaskThreadLocalStorage local_tls;
#ifndef NDEBUG
pthread_t creator_thread_id;
#endif

#ifdef DEBUG_STATS
TaskMemPoolStats *mempool_stats;
#endif
};		};

struct TaskScheduler {		struct TaskScheduler {
pthread_t *threads;		pthread_t *threads;
struct TaskThread *task_threads;		struct TaskThread *task_threads;
int num_threads;		int num_threads;
bool background_thread_only;		bool background_thread_only;

Show All 9 Lines	struct TaskScheduler {

/* NOTE: In pthread's TLS we store the whole TaskThread structure. */		/* NOTE: In pthread's TLS we store the whole TaskThread structure. */
pthread_key_t tls_id_key;		pthread_key_t tls_id_key;
};		};

typedef struct TaskThread {		typedef struct TaskThread {
TaskScheduler *scheduler;		TaskScheduler *scheduler;
int id;		int id;
TaskThreadLocalStorage tls;
} TaskThread;		} TaskThread;

/* Helper */		/* Helper */
BLI_INLINE void task_data_free(Task *task, const int thread_id)		BLI_INLINE void task_data_free(Task *task)
{		{
if (task->free_taskdata) {		if (task->free_taskdata) {
if (task->freedata) {		if (task->freedata) {
task->freedata(task->pool, task->taskdata, thread_id);		task->freedata(task->pool, task->taskdata);
}		}
else {		else {
MEM_freeN(task->taskdata);		MEM_freeN(task->taskdata);
}		}
}		}
}		}

BLI_INLINE void initialize_task_tls(TaskThreadLocalStorage *tls)		static Task task_alloc(TaskPool UNUSED(pool))
{
memset(tls, 0, sizeof(TaskThreadLocalStorage));
}

BLI_INLINE TaskThreadLocalStorage get_task_tls(TaskPool pool, const int thread_id)
{
TaskScheduler *scheduler = pool->scheduler;
BLI_assert(thread_id >= 0);
BLI_assert(thread_id <= scheduler->num_threads);
if (pool->use_local_tls && thread_id == 0) {
BLI_assert(pool->thread_id == 0);
BLI_assert(!BLI_thread_is_main());
BLI_assert(pthread_equal(pthread_self(), pool->creator_thread_id));
return &pool->local_tls;
}
if (thread_id == 0) {
BLI_assert(BLI_thread_is_main());
return &scheduler->task_threads[pool->thread_id].tls;
}
return &scheduler->task_threads[thread_id].tls;
}

BLI_INLINE void free_task_tls(TaskThreadLocalStorage *tls)
{
TaskMemPool *task_mempool = &tls->task_mempool;
for (int i = 0; i < task_mempool->num_tasks; i++) {
MEM_freeN(task_mempool->tasks[i]);
}
}

static Task task_alloc(TaskPool pool, const int thread_id)
{		{
BLI_assert(thread_id <= pool->scheduler->num_threads);
if (thread_id != -1) {
BLI_assert(thread_id >= 0);
BLI_assert(thread_id <= pool->scheduler->num_threads);
TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
TaskMemPool *task_mempool = &tls->task_mempool;
/* Try to re-use task memory from a thread local storage. */
if (task_mempool->num_tasks > 0) {
--task_mempool->num_tasks;
/* Success! We've just avoided task allocation. */
#ifdef DEBUG_STATS
pool->mempool_stats[thread_id].num_reuse++;
#endif
return task_mempool->tasks[task_mempool->num_tasks];
}
/* We are doomed to allocate new task data. */
#ifdef DEBUG_STATS
pool->mempool_stats[thread_id].num_alloc++;
#endif
}
return (Task *)MEM_mallocN(sizeof(Task), "New task");		return (Task *)MEM_mallocN(sizeof(Task), "New task");
}		}

static void task_free(TaskPool pool, Task task, const int thread_id)		static void task_free(TaskPool UNUSED(pool), Task task)
{		{
task_data_free(task, thread_id);		task_data_free(task);
BLI_assert(thread_id >= 0);
BLI_assert(thread_id <= pool->scheduler->num_threads);
if (thread_id == 0) {
BLI_assert(pool->use_local_tls \|\| BLI_thread_is_main());
}
TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
TaskMemPool *task_mempool = &tls->task_mempool;
if (task_mempool->num_tasks < MEMPOOL_SIZE - 1) {
/* Successfully allowed the task to be re-used later. */
task_mempool->tasks[task_mempool->num_tasks] = task;
++task_mempool->num_tasks;
}
else {
/* Local storage saturated, no other way than just discard
* the memory.
*
* TODO(sergey): We can perhaps store such pointer in a global
* scheduler pool, maybe it'll be faster than discarding and
* allocating again.
*/
MEM_freeN(task);		MEM_freeN(task);
#ifdef DEBUG_STATS
pool->mempool_stats[thread_id].num_discard++;
#endif
}
}		}

/* Task Scheduler */		/* Task Scheduler */

static void task_pool_num_decrease(TaskPool *pool, size_t done)		static void task_pool_num_decrease(TaskPool *pool, size_t done)
{		{
BLI_mutex_lock(&pool->num_mutex);		BLI_mutex_lock(&pool->num_mutex);

▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines	do {
}		}
} while (!found_task);		} while (!found_task);

BLI_mutex_unlock(&scheduler->queue_mutex);		BLI_mutex_unlock(&scheduler->queue_mutex);

return true;		return true;
}		}

BLI_INLINE void handle_local_queue(TaskThreadLocalStorage *tls, const int thread_id)
{
BLI_assert(!tls->do_delayed_push);
while (tls->num_local_queue > 0) {
/* We pop task from queue before handling it so handler of the task can
* push next job to the local queue.
*/
tls->num_local_queue--;
Task *local_task = tls->local_queue[tls->num_local_queue];
/* TODO(sergey): Double-check work_and_wait() doesn't handle other's
* pool tasks.
*/
TaskPool *local_pool = local_task->pool;
local_task->run(local_pool, local_task->taskdata, thread_id);
task_free(local_pool, local_task, thread_id);
}
BLI_assert(!tls->do_delayed_push);
}

static void task_scheduler_thread_run(void thread_p)		static void task_scheduler_thread_run(void thread_p)
{		{
TaskThread thread = (TaskThread )thread_p;		TaskThread thread = (TaskThread )thread_p;
TaskThreadLocalStorage *tls = &thread->tls;
TaskScheduler *scheduler = thread->scheduler;		TaskScheduler *scheduler = thread->scheduler;
int thread_id = thread->id;
Task *task;		Task *task;

pthread_setspecific(scheduler->tls_id_key, thread);		pthread_setspecific(scheduler->tls_id_key, thread);

/* signal the main thread when all threads have started */		/* signal the main thread when all threads have started */
BLI_mutex_lock(&scheduler->startup_mutex);		BLI_mutex_lock(&scheduler->startup_mutex);
scheduler->num_thread_started++;		scheduler->num_thread_started++;
if (scheduler->num_thread_started == scheduler->num_threads) {		if (scheduler->num_thread_started == scheduler->num_threads) {
BLI_condition_notify_one(&scheduler->startup_cond);		BLI_condition_notify_one(&scheduler->startup_cond);
}		}
BLI_mutex_unlock(&scheduler->startup_mutex);		BLI_mutex_unlock(&scheduler->startup_mutex);

/* keep popping off tasks */		/* keep popping off tasks */
while (task_scheduler_thread_wait_pop(scheduler, &task)) {		while (task_scheduler_thread_wait_pop(scheduler, &task)) {
TaskPool *pool = task->pool;		TaskPool *pool = task->pool;

/* run task */		/* run task */
BLI_assert(!tls->do_delayed_push);		task->run(pool, task->taskdata);
task->run(pool, task->taskdata, thread_id);
BLI_assert(!tls->do_delayed_push);

/* delete task */		/* delete task */
task_free(pool, task, thread_id);		task_free(pool, task);

/* Handle all tasks from local queue. */
handle_local_queue(tls, thread_id);

/* notify pool task was done */		/* notify pool task was done */
task_pool_num_decrease(pool, 1);		task_pool_num_decrease(pool, 1);
}		}

return NULL;		return NULL;
}		}

TaskScheduler *BLI_task_scheduler_create(int num_threads)		TaskScheduler *BLI_task_scheduler_legacy_create(int num_threads)
{		{
TaskScheduler scheduler = (TaskScheduler )MEM_callocN(sizeof(TaskScheduler), "TaskScheduler");		TaskScheduler scheduler = (TaskScheduler )MEM_callocN(sizeof(TaskScheduler), "TaskScheduler");

/* multiple places can use this task scheduler, sharing the same		/* multiple places can use this task scheduler, sharing the same
* threads, so we keep track of the number of users. */		* threads, so we keep track of the number of users. */
scheduler->do_exit = false;		scheduler->do_exit = false;

BLI_listbase_clear(&scheduler->queue);		BLI_listbase_clear(&scheduler->queue);
Show All 17 Lines	if (num_threads == 0) {
scheduler->background_thread_only = true;		scheduler->background_thread_only = true;
num_threads = 1;		num_threads = 1;
}		}

scheduler->task_threads = (TaskThread )MEM_mallocN(sizeof(TaskThread) (num_threads + 1),		scheduler->task_threads = (TaskThread )MEM_mallocN(sizeof(TaskThread) (num_threads + 1),
"TaskScheduler task threads");		"TaskScheduler task threads");

/* Initialize TLS for main thread. */		/* Initialize TLS for main thread. */
initialize_task_tls(&scheduler->task_threads[0].tls);

pthread_key_create(&scheduler->tls_id_key, NULL);		pthread_key_create(&scheduler->tls_id_key, NULL);

/* launch threads that will be waiting for work */		/* launch threads that will be waiting for work */
if (num_threads > 0) {		if (num_threads > 0) {
int i;		int i;

scheduler->num_threads = num_threads;		scheduler->num_threads = num_threads;
scheduler->threads = (pthread_t )MEM_callocN(sizeof(pthread_t) num_threads,		scheduler->threads = (pthread_t )MEM_callocN(sizeof(pthread_t) num_threads,
"TaskScheduler threads");		"TaskScheduler threads");

for (i = 0; i < num_threads; i++) {		for (i = 0; i < num_threads; i++) {
TaskThread *thread = &scheduler->task_threads[i + 1];		TaskThread *thread = &scheduler->task_threads[i + 1];
thread->scheduler = scheduler;		thread->scheduler = scheduler;
thread->id = i + 1;		thread->id = i + 1;
initialize_task_tls(&thread->tls);

if (pthread_create(&scheduler->threads[i], NULL, task_scheduler_thread_run, thread) != 0) {		if (pthread_create(&scheduler->threads[i], NULL, task_scheduler_thread_run, thread) != 0) {
fprintf(stderr, "TaskScheduler failed to launch thread %d/%d\n", i, num_threads);		fprintf(stderr, "TaskScheduler failed to launch thread %d/%d\n", i, num_threads);
}		}
}		}
}		}

/* Wait for all worker threads to start before returning to caller to prevent the case where		/* Wait for all worker threads to start before returning to caller to prevent the case where
* threads are still starting and pthread_join is called, which causes a deadlock on pthreads4w.		* threads are still starting and pthread_join is called, which causes a deadlock on
		pthreads4w.
*/		*/
BLI_mutex_lock(&scheduler->startup_mutex);		BLI_mutex_lock(&scheduler->startup_mutex);
/* NOTE: Use loop here to avoid false-positive everything-is-ready caused by spontaneous thread		/* NOTE: Use loop here to avoid false-positive everything-is-ready caused by spontaneous
		thread
* wake up. */		* wake up. */
while (scheduler->num_thread_started != num_threads) {		while (scheduler->num_thread_started != num_threads) {
BLI_condition_wait(&scheduler->startup_cond, &scheduler->startup_mutex);		BLI_condition_wait(&scheduler->startup_cond, &scheduler->startup_mutex);
}		}
BLI_mutex_unlock(&scheduler->startup_mutex);		BLI_mutex_unlock(&scheduler->startup_mutex);

return scheduler;		return scheduler;
}		}

void BLI_task_scheduler_free(TaskScheduler *scheduler)
{
Task *task;

/* stop all waiting threads */
BLI_mutex_lock(&scheduler->queue_mutex);
scheduler->do_exit = true;
BLI_condition_notify_all(&scheduler->queue_cond);
BLI_mutex_unlock(&scheduler->queue_mutex);

pthread_key_delete(scheduler->tls_id_key);

/* delete threads */
if (scheduler->threads) {
int i;

for (i = 0; i < scheduler->num_threads; i++) {
if (pthread_join(scheduler->threads[i], NULL) != 0) {
fprintf(stderr, "TaskScheduler failed to join thread %d/%d\n", i, scheduler->num_threads);
}
}

MEM_freeN(scheduler->threads);
}

/* Delete task thread data */
if (scheduler->task_threads) {
for (int i = 0; i < scheduler->num_threads + 1; i++) {
TaskThreadLocalStorage *tls = &scheduler->task_threads[i].tls;
free_task_tls(tls);
}

MEM_freeN(scheduler->task_threads);
}

/* delete leftover tasks */
for (task = (Task *)scheduler->queue.first; task; task = task->next) {
task_data_free(task, 0);
}
BLI_freelistN(&scheduler->queue);

/* delete mutex/condition */
BLI_mutex_end(&scheduler->queue_mutex);
BLI_condition_end(&scheduler->queue_cond);
BLI_mutex_end(&scheduler->startup_mutex);
BLI_condition_end(&scheduler->startup_cond);

MEM_freeN(scheduler);
}

int BLI_task_scheduler_num_threads(TaskScheduler *scheduler)
{
return scheduler->num_threads + 1;
}

static void task_scheduler_push(TaskScheduler scheduler, Task task, TaskPriority priority)		static void task_scheduler_push(TaskScheduler scheduler, Task task, TaskPriority priority)
{		{
task_pool_num_increase(task->pool, 1);		task_pool_num_increase(task->pool, 1);

/* add task to queue */		/* add task to queue */
BLI_mutex_lock(&scheduler->queue_mutex);		BLI_mutex_lock(&scheduler->queue_mutex);

if (priority == TASK_PRIORITY_HIGH) {		if (priority == TASK_PRIORITY_HIGH) {
BLI_addhead(&scheduler->queue, task);		BLI_addhead(&scheduler->queue, task);
}		}
else {		else {
BLI_addtail(&scheduler->queue, task);		BLI_addtail(&scheduler->queue, task);
}		}

BLI_condition_notify_one(&scheduler->queue_cond);		BLI_condition_notify_one(&scheduler->queue_cond);
BLI_mutex_unlock(&scheduler->queue_mutex);		BLI_mutex_unlock(&scheduler->queue_mutex);
}		}

static void task_scheduler_push_all(TaskScheduler *scheduler,
TaskPool *pool,
Task **tasks,
int num_tasks)
{
if (num_tasks == 0) {
return;
}

task_pool_num_increase(pool, num_tasks);

BLI_mutex_lock(&scheduler->queue_mutex);

for (int i = 0; i < num_tasks; i++) {
BLI_addhead(&scheduler->queue, tasks[i]);
}

BLI_condition_notify_all(&scheduler->queue_cond);
BLI_mutex_unlock(&scheduler->queue_mutex);
}

static void task_scheduler_clear(TaskScheduler scheduler, TaskPool pool)		static void task_scheduler_clear(TaskScheduler scheduler, TaskPool pool)
{		{
Task task, nexttask;		Task task, nexttask;
size_t done = 0;		size_t done = 0;

BLI_mutex_lock(&scheduler->queue_mutex);		BLI_mutex_lock(&scheduler->queue_mutex);

/* free all tasks from this pool from the queue */		/* free all tasks from this pool from the queue */
for (task = (Task *)scheduler->queue.first; task; task = nexttask) {		for (task = (Task *)scheduler->queue.first; task; task = nexttask) {
nexttask = task->next;		nexttask = task->next;

if (task->pool == pool) {		if (task->pool == pool) {
task_data_free(task, pool->thread_id);		task_data_free(task);
BLI_freelinkN(&scheduler->queue, task);		BLI_freelinkN(&scheduler->queue, task);

done++;		done++;
}		}
}		}

BLI_mutex_unlock(&scheduler->queue_mutex);		BLI_mutex_unlock(&scheduler->queue_mutex);

Show All 29 Lines	#endif
pool->do_cancel = false;		pool->do_cancel = false;
pool->do_work = false;		pool->do_work = false;
pool->is_suspended = is_suspended;		pool->is_suspended = is_suspended;
pool->start_suspended = is_suspended;		pool->start_suspended = is_suspended;
pool->num_suspended = 0;		pool->num_suspended = 0;
pool->suspended_queue.first = pool->suspended_queue.last = NULL;		pool->suspended_queue.first = pool->suspended_queue.last = NULL;
pool->priority = priority;		pool->priority = priority;
pool->run_in_background = is_background;		pool->run_in_background = is_background;
pool->use_local_tls = false;

BLI_mutex_init(&pool->num_mutex);		BLI_mutex_init(&pool->num_mutex);
BLI_condition_init(&pool->num_cond);		BLI_condition_init(&pool->num_cond);

pool->userdata = userdata;		pool->userdata = userdata;
BLI_mutex_init(&pool->user_mutex);		BLI_mutex_init(&pool->user_mutex);

if (BLI_thread_is_main()) {		if (BLI_thread_is_main()) {
pool->thread_id = 0;		pool->thread_id = 0;
}		}
else {		else {
TaskThread thread = (TaskThread )pthread_getspecific(scheduler->tls_id_key);		TaskThread thread = (TaskThread )pthread_getspecific(scheduler->tls_id_key);
if (thread == NULL) {		if (thread == NULL) {
/* NOTE: Task pool is created from non-main thread which is not		/* NOTE: Task pool is created from non-main thread which is not
* managed by the task scheduler. We identify ourselves as thread ID		* managed by the task scheduler. We identify ourselves as thread ID
* 0 but we do not use scheduler's TLS storage and use our own		* 0 but we do not use scheduler's TLS storage and use our own
* instead to avoid any possible threading conflicts.		* instead to avoid any possible threading conflicts.
*/		*/
pool->thread_id = 0;		pool->thread_id = 0;
pool->use_local_tls = true;
#ifndef NDEBUG
pool->creator_thread_id = pthread_self();
#endif
initialize_task_tls(&pool->local_tls);
}		}
else {		else {
pool->thread_id = thread->id;		pool->thread_id = thread->id;
}		}
}		}

#ifdef DEBUG_STATS
pool->mempool_stats = (TaskMemPoolStats *)MEM_callocN(
sizeof(pool->mempool_stats) (scheduler->num_threads + 1), "per-taskpool mempool stats");
#endif

/* Ensure malloc will go fine from threads,		/* Ensure malloc will go fine from threads,
*		*
* This is needed because we could be in main thread here		* This is needed because we could be in main thread here
* and malloc could be non-thread safe at this point because		* and malloc could be non-thread safe at this point because
* no other jobs are running.		* no other jobs are running.
*/		*/
BLI_threaded_malloc_begin();		BLI_threaded_malloc_begin();

return pool;		return pool;
}		}

/**		/**
* Create a normal task pool. Tasks will be executed as soon as they are added.		* Create a normal task pool. Tasks will be executed as soon as they are added.
*/		*/
TaskPool BLI_task_pool_create(TaskScheduler scheduler, void *userdata, TaskPriority priority)		TaskPool BLI_task_pool_legacy_create(TaskScheduler scheduler,
		void *userdata,
		TaskPriority priority)
{		{
return task_pool_create_ex(scheduler, userdata, false, false, priority);		return task_pool_create_ex(scheduler, userdata, false, false, priority);
}		}

/**		/**
* Create a background task pool.		* Create a background task pool.
* In multi-threaded context, there is no differences with #BLI_task_pool_create(),		* In multi-threaded context, there is no differences with #BLI_task_pool_legacy_create(),
* but in single-threaded case it is ensured to have at least one worker thread to run on		* but in single-threaded case it is ensured to have at least one worker thread to run on
* (i.e. you don't have to call #BLI_task_pool_work_and_wait		* (i.e. you don't have to call #BLI_task_pool_legacy_work_and_wait
* on it to be sure it will be processed).		* on it to be sure it will be processed).
*		*
* \note Background pools are non-recursive		* \note Background pools are non-recursive
* (that is, you should not create other background pools in tasks assigned to a background pool,		* (that is, you should not create other background pools in tasks assigned to a background
		pool,
* they could end never being executed, since the 'fallback' background thread is already		* they could end never being executed, since the 'fallback' background thread is already
* busy with parent task in single-threaded context).		* busy with parent task in single-threaded context).
*/		*/
TaskPool BLI_task_pool_create_background(TaskScheduler scheduler,		TaskPool BLI_task_pool_legacy_create_background(TaskScheduler scheduler,
void *userdata,		void *userdata,
TaskPriority priority)		TaskPriority priority)
{		{
return task_pool_create_ex(scheduler, userdata, true, false, priority);		return task_pool_create_ex(scheduler, userdata, true, false, priority);
}		}

/**		/**
* Similar to BLI_task_pool_create() but does not schedule any tasks for execution		* Similar to BLI_task_pool_legacy_create() but does not schedule any tasks for execution
* for until BLI_task_pool_work_and_wait() is called. This helps reducing threading		* for until BLI_task_pool_legacy_work_and_wait() is called. This helps reducing threading
* overhead when pushing huge amount of small initial tasks from the main thread.		* overhead when pushing huge amount of small initial tasks from the main thread.
*/		*/
TaskPool BLI_task_pool_create_suspended(TaskScheduler scheduler,		TaskPool BLI_task_pool_legacy_create_suspended(TaskScheduler scheduler,
void *userdata,		void *userdata,
TaskPriority priority)		TaskPriority priority)
{		{
return task_pool_create_ex(scheduler, userdata, false, true, priority);		return task_pool_create_ex(scheduler, userdata, false, true, priority);
}		}

void BLI_task_pool_free(TaskPool *pool)		void BLI_task_pool_legacy_free(TaskPool *pool)
{		{
BLI_task_pool_cancel(pool);		BLI_task_pool_legacy_cancel(pool);

BLI_mutex_end(&pool->num_mutex);		BLI_mutex_end(&pool->num_mutex);
BLI_condition_end(&pool->num_cond);		BLI_condition_end(&pool->num_cond);

BLI_mutex_end(&pool->user_mutex);		BLI_mutex_end(&pool->user_mutex);

#ifdef DEBUG_STATS
printf("Thread ID Allocated Reused Discarded\n");
for (int i = 0; i < pool->scheduler->num_threads + 1; i++) {
printf("%02d %05d %05d %05d\n",
i,
pool->mempool_stats[i].num_alloc,
pool->mempool_stats[i].num_reuse,
pool->mempool_stats[i].num_discard);
}
MEM_freeN(pool->mempool_stats);
#endif

if (pool->use_local_tls) {
free_task_tls(&pool->local_tls);
}

MEM_freeN(pool);		MEM_freeN(pool);

BLI_threaded_malloc_end();		BLI_threaded_malloc_end();
}		}

BLI_INLINE bool task_can_use_local_queues(TaskPool *pool, int thread_id)		void BLI_task_pool_legacy_push(TaskPool *pool,
{
return (thread_id != -1 && (thread_id != pool->thread_id \|\| pool->do_work));
}

static void task_pool_push(TaskPool *pool,
TaskRunFunction run,		TaskRunFunction run,
void *taskdata,		void *taskdata,
bool free_taskdata,		bool free_taskdata,
TaskFreeFunction freedata,		TaskFreeFunction freedata)
int thread_id)
{		{
/* Allocate task and fill it's properties. */		/* Allocate task and fill it's properties. */
Task *task = task_alloc(pool, thread_id);		Task *task = task_alloc(pool);
task->run = run;		task->run = run;
task->taskdata = taskdata;		task->taskdata = taskdata;
task->free_taskdata = free_taskdata;		task->free_taskdata = free_taskdata;
task->freedata = freedata;		task->freedata = freedata;
task->pool = pool;		task->pool = pool;
/* For suspended pools we put everything yo a global queue first		/* For suspended pools we put everything yo a global queue first
* and exit as soon as possible.		* and exit as soon as possible.
*		*
* This tasks will be moved to actual execution when pool is		* This tasks will be moved to actual execution when pool is
* activated by work_and_wait().		* activated by work_and_wait().
*/		*/
if (pool->is_suspended) {		if (pool->is_suspended) {
BLI_addhead(&pool->suspended_queue, task);		BLI_addhead(&pool->suspended_queue, task);
atomic_fetch_and_add_z(&pool->num_suspended, 1);		atomic_fetch_and_add_z(&pool->num_suspended, 1);
return;		return;
}		}
/* Populate to any local queue first, this is cheapest push ever. */
if (task_can_use_local_queues(pool, thread_id)) {
ASSERT_THREAD_ID(pool->scheduler, thread_id);
TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
/* Try to push to a local execution queue.
* These tasks will be picked up next.
*/
if (tls->num_local_queue < LOCAL_QUEUE_SIZE) {
tls->local_queue[tls->num_local_queue] = task;
tls->num_local_queue++;
return;
}
/* If we are in the delayed tasks push mode, we push tasks to a
* temporary local queue first without any locks, and then move them
* to global execution queue with a single lock.
*/
if (tls->do_delayed_push && tls->num_delayed_queue < DELAYED_QUEUE_SIZE) {
tls->delayed_queue[tls->num_delayed_queue] = task;
tls->num_delayed_queue++;
return;
}
}
/* Do push to a global execution pool, slowest possible method,		/* Do push to a global execution pool, slowest possible method,
* causes quite reasonable amount of threading overhead.		* causes quite reasonable amount of threading overhead.
*/		*/
task_scheduler_push(pool->scheduler, task, pool->priority);		task_scheduler_push(pool->scheduler, task, pool->priority);
}		}

void BLI_task_pool_push(TaskPool *pool,		void BLI_task_pool_legacy_work_and_wait(TaskPool *pool)
TaskRunFunction run,
void *taskdata,
bool free_taskdata,
TaskFreeFunction freedata)
{
task_pool_push(pool, run, taskdata, free_taskdata, freedata, -1);
}

void BLI_task_pool_push_from_thread(TaskPool *pool,
TaskRunFunction run,
void *taskdata,
bool free_taskdata,
TaskFreeFunction freedata,
int thread_id)
{
task_pool_push(pool, run, taskdata, free_taskdata, freedata, thread_id);
}

void BLI_task_pool_work_and_wait(TaskPool *pool)
{		{
TaskThreadLocalStorage *tls = get_task_tls(pool, pool->thread_id);
TaskScheduler *scheduler = pool->scheduler;		TaskScheduler *scheduler = pool->scheduler;

if (atomic_fetch_and_and_uint8((uint8_t *)&pool->is_suspended, 0)) {		if (atomic_fetch_and_and_uint8((uint8_t *)&pool->is_suspended, 0)) {
if (pool->num_suspended) {		if (pool->num_suspended) {
task_pool_num_increase(pool, pool->num_suspended);		task_pool_num_increase(pool, pool->num_suspended);
BLI_mutex_lock(&scheduler->queue_mutex);		BLI_mutex_lock(&scheduler->queue_mutex);

BLI_movelisttolist(&scheduler->queue, &pool->suspended_queue);		BLI_movelisttolist(&scheduler->queue, &pool->suspended_queue);

BLI_condition_notify_all(&scheduler->queue_cond);		BLI_condition_notify_all(&scheduler->queue_cond);
BLI_mutex_unlock(&scheduler->queue_mutex);		BLI_mutex_unlock(&scheduler->queue_mutex);

pool->num_suspended = 0;		pool->num_suspended = 0;
}		}
}		}

pool->do_work = true;		pool->do_work = true;

ASSERT_THREAD_ID(pool->scheduler, pool->thread_id);		ASSERT_THREAD_ID(pool->scheduler, pool->thread_id);

handle_local_queue(tls, pool->thread_id);

BLI_mutex_lock(&pool->num_mutex);		BLI_mutex_lock(&pool->num_mutex);

while (pool->num != 0) {		while (pool->num != 0) {
Task task, work_task = NULL;		Task task, work_task = NULL;
bool found_task = false;		bool found_task = false;

BLI_mutex_unlock(&pool->num_mutex);		BLI_mutex_unlock(&pool->num_mutex);

Show All 11 Lines	for (task = (Task *)scheduler->queue.first; task; task = task->next) {
}		}
}		}

BLI_mutex_unlock(&scheduler->queue_mutex);		BLI_mutex_unlock(&scheduler->queue_mutex);

/* if found task, do it, otherwise wait until other tasks are done */		/* if found task, do it, otherwise wait until other tasks are done */
if (found_task) {		if (found_task) {
/* run task */		/* run task */
BLI_assert(!tls->do_delayed_push);		work_task->run(pool, work_task->taskdata);
work_task->run(pool, work_task->taskdata, pool->thread_id);
BLI_assert(!tls->do_delayed_push);

/* delete task */		/* delete task */
task_free(pool, task, pool->thread_id);		task_free(pool, task);

/* Handle all tasks from local queue. */
handle_local_queue(tls, pool->thread_id);

/* notify pool task was done */		/* notify pool task was done */
task_pool_num_decrease(pool, 1);		task_pool_num_decrease(pool, 1);
}		}

BLI_mutex_lock(&pool->num_mutex);		BLI_mutex_lock(&pool->num_mutex);
if (pool->num == 0) {		if (pool->num == 0) {
break;		break;
}		}

if (!found_task) {		if (!found_task) {
BLI_condition_wait(&pool->num_cond, &pool->num_mutex);		BLI_condition_wait(&pool->num_cond, &pool->num_mutex);
}		}
}		}

BLI_mutex_unlock(&pool->num_mutex);		BLI_mutex_unlock(&pool->num_mutex);

BLI_assert(tls->num_local_queue == 0);
}		}

void BLI_task_pool_work_wait_and_reset(TaskPool *pool)		void BLI_task_pool_legacy_work_wait_and_reset(TaskPool *pool)
{		{
BLI_task_pool_work_and_wait(pool);		BLI_task_pool_legacy_work_and_wait(pool);

pool->do_work = false;		pool->do_work = false;
pool->is_suspended = pool->start_suspended;		pool->is_suspended = pool->start_suspended;
}		}

void BLI_task_pool_cancel(TaskPool *pool)		void BLI_task_pool_legacy_cancel(TaskPool *pool)
{		{
pool->do_cancel = true;		pool->do_cancel = true;

task_scheduler_clear(pool->scheduler, pool);		task_scheduler_clear(pool->scheduler, pool);

/* wait until all entries are cleared */		/* wait until all entries are cleared */
BLI_mutex_lock(&pool->num_mutex);		BLI_mutex_lock(&pool->num_mutex);
while (pool->num) {		while (pool->num) {
BLI_condition_wait(&pool->num_cond, &pool->num_mutex);		BLI_condition_wait(&pool->num_cond, &pool->num_mutex);
}		}
BLI_mutex_unlock(&pool->num_mutex);		BLI_mutex_unlock(&pool->num_mutex);

pool->do_cancel = false;		pool->do_cancel = false;
}		}

bool BLI_task_pool_canceled(TaskPool *pool)		bool BLI_task_pool_legacy_canceled(TaskPool *pool)
{		{
return pool->do_cancel;		return pool->do_cancel;
}		}

void BLI_task_pool_userdata(TaskPool pool)		void BLI_task_pool_legacy_user_data(TaskPool pool)
{		{
return pool->userdata;		return pool->userdata;
}		}

ThreadMutex BLI_task_pool_user_mutex(TaskPool pool)		ThreadMutex BLI_task_pool_legacy_user_mutex(TaskPool pool)
{		{
return &pool->user_mutex;		return &pool->user_mutex;
}		}

int BLI_task_pool_creator_thread_id(TaskPool *pool)
{
return pool->thread_id;
}

void BLI_task_pool_delayed_push_begin(TaskPool *pool, int thread_id)
{
if (task_can_use_local_queues(pool, thread_id)) {
ASSERT_THREAD_ID(pool->scheduler, thread_id);
TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
tls->do_delayed_push = true;
}
}

void BLI_task_pool_delayed_push_end(TaskPool *pool, int thread_id)
{
if (task_can_use_local_queues(pool, thread_id)) {
ASSERT_THREAD_ID(pool->scheduler, thread_id);
TaskThreadLocalStorage *tls = get_task_tls(pool, thread_id);
BLI_assert(tls->do_delayed_push);
task_scheduler_push_all(pool->scheduler, pool, tls->delayed_queue, tls->num_delayed_queue);
tls->do_delayed_push = false;
tls->num_delayed_queue = 0;
}
}