/*
 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/*
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 *
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *	File:	sched_prim.h
 *	Author:	David Golub
 *
 *	Scheduling primitive definitions file
 *
 */

#ifndef _KERN_SCHED_PRIM_H_
#define _KERN_SCHED_PRIM_H_

#include <sys/cdefs.h>
#include <mach/boolean.h>
#include <mach/machine/vm_types.h>
#include <mach/kern_return.h>
#include <kern/clock.h>
#include <kern/kern_types.h>
#include <kern/percpu.h>
#include <kern/thread.h>
#include <kern/block_hint.h>

extern int              thread_get_current_cpuid(void);

#if XNU_KERNEL_PRIVATE

/*
 * The quantum length used for Fixed and RT sched modes. In general the quantum
 * can vary - for example for background or QOS.
 */
uint64_t sched_get_quantum_us(void);

#endif /* XNU_KERNEL_PRIVATE */

#if defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS

#include <kern/sched_urgency.h>
#include <kern/thread_group.h>
#include <kern/waitq.h>

/* Initialization */
extern void             sched_init(void);

extern void             sched_startup(void);

extern void             sched_timebase_init(void);

extern bool             processor_is_fast_track_candidate_for_realtime_thread(processor_set_t pset, processor_t processor);

extern void             sched_check_spill(processor_set_t pset, thread_t thread);

extern bool             sched_thread_should_yield(processor_t processor, thread_t thread);

extern bool             sched_steal_thread_DISABLED(processor_set_t pset);
extern bool             sched_steal_thread_enabled(processor_set_t pset);

/* Force a preemption point for a thread and wait for it to stop running */
extern boolean_t        thread_stop(
	thread_t        thread,
	boolean_t       until_not_runnable);

/* Release a previous stop request */
extern void                     thread_unstop(
	thread_t        thread);

/* Wait for a thread to stop running */
extern void                     thread_wait(
	thread_t        thread,
	boolean_t       until_not_runnable);

/* Unblock thread on wake up */
extern boolean_t        thread_unblock(
	thread_t                thread,
	wait_result_t   wresult);

/* Unblock and dispatch thread */
extern void thread_go(
	thread_t                thread,
	wait_result_t           wresult,
	bool                    try_handoff);

/* Check if direct handoff is allowed */
extern boolean_t
thread_allowed_for_handoff(
	thread_t         thread);

/* Handle threads at context switch */
extern void                     thread_dispatch(
	thread_t                old_thread,
	thread_t                new_thread);

/* Switch directly to a particular thread */
extern int                      thread_run(
	thread_t                        self,
	thread_continue_t       continuation,
	void                            *parameter,
	thread_t                        new_thread);

/* Resume thread with new stack */
extern __dead2 void     thread_continue(thread_t old_thread);

/* Invoke continuation */
extern __dead2 void     call_continuation(
	thread_continue_t       continuation,
	void                    *parameter,
	wait_result_t           wresult,
	boolean_t               enable_interrupts);

/*
 * Flags that can be passed to set_sched_pri
 * to skip side effects
 */
__options_decl(set_sched_pri_options_t, uint32_t, {
	SETPRI_DEFAULT  = 0x0,
	SETPRI_LAZY     = 0x1,  /* Avoid setting AST flags or sending IPIs */
});

/* Set the current scheduled priority */
extern void set_sched_pri(
	thread_t      thread,
	int16_t       priority,
	set_sched_pri_options_t options);

/* Set base priority of the specified thread */
extern void             sched_set_thread_base_priority(
	thread_t                thread,
	int                             priority);

/* Set absolute base priority of the specified thread */
extern void             sched_set_kernel_thread_priority(
	thread_t                thread,
	int                             priority);


/* Set the thread's true scheduling mode */
extern void             sched_set_thread_mode(thread_t thread,
    sched_mode_t mode);

/*
 * Set the thread's scheduling mode taking into account that the thread may have
 * been demoted.
 * */
extern void             sched_set_thread_mode_user(thread_t thread,
    sched_mode_t mode);

/*
 * Get the thread's scheduling mode taking into account that the thread may have
 * been demoted.
 * */
extern sched_mode_t     sched_get_thread_mode_user(thread_t thread);


/* Demote the true scheduler mode */
extern void             sched_thread_mode_demote(thread_t thread,
    uint32_t reason);
/* Un-demote the true scheduler mode */
extern void             sched_thread_mode_undemote(thread_t thread,
    uint32_t reason);
/* Check for a specific demotion */
extern bool             sched_thread_mode_has_demotion(thread_t thread,
    uint32_t reason);

extern void sched_thread_promote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj);
extern void sched_thread_unpromote_reason(thread_t thread, uint32_t reason, uintptr_t trace_obj);

/* Re-evaluate base priority of thread (thread locked) */
void thread_recompute_priority(thread_t thread);

/* Re-evaluate scheduled priority of thread (thread locked) */
extern void thread_recompute_sched_pri(
	thread_t thread,
	set_sched_pri_options_t options);

/* Periodic scheduler activity */
extern void             sched_init_thread(void);

/* Perform sched_tick housekeeping activities */
extern boolean_t                can_update_priority(
	thread_t                thread);

extern void             update_priority(
	thread_t                thread);

extern void             lightweight_update_priority(
	thread_t                thread);

extern void             sched_default_quantum_expire(thread_t thread);

/* Idle processor thread continuation */
extern void             idle_thread(
	void*           parameter,
	wait_result_t   result);

extern void idle_thread_create(
	processor_t             processor,
	thread_continue_t       continuation);

/* Continuation return from syscall */
extern void     thread_syscall_return(
	kern_return_t   ret);

/* Context switch */
extern wait_result_t    thread_block_reason(
	thread_continue_t       continuation,
	void                            *parameter,
	ast_t                           reason);

__options_decl(sched_options_t, uint32_t, {
	SCHED_NONE      = 0x0,
	SCHED_TAILQ     = 0x1,
	SCHED_HEADQ     = 0x2,
	SCHED_PREEMPT   = 0x4,
	SCHED_REBALANCE = 0x8,
	SCHED_STIR_POT  = 0x10,
});

/* Reschedule thread for execution */
extern void             thread_setrun(
	thread_t        thread,
	sched_options_t options);

extern processor_set_t  task_choose_pset(
	task_t                  task);

/* Bind the current thread to a particular processor */
extern processor_t      thread_bind(
	processor_t             processor);

extern void             thread_bind_during_wakeup(
	thread_t                thread,
	processor_t             processor);

extern void             thread_unbind_after_queue_shutdown(
	thread_t                thread,
	processor_t             processor);

extern bool pset_has_stealable_threads(
	processor_set_t         pset);

extern processor_set_t choose_starting_pset(
	pset_node_t  node,
	thread_t     thread,
	processor_t *processor_hint);

extern int pset_available_cpu_count(
	processor_set_t pset);

extern bool pset_is_recommended(
	processor_set_t pset);

extern bool pset_type_is_recommended(
	processor_set_t pset);

extern pset_node_t sched_choose_node(
	thread_t     thread);

#if CONFIG_SCHED_SMT
extern processor_t      choose_processor_smt(
	processor_set_t                pset,
	processor_t                    processor,
	thread_t                       thread,
	sched_options_t               *options);
#else /* !CONFIG_SCHED_SMT */
/* Choose the best processor to run a thread */
extern processor_t      choose_processor(
	processor_set_t                pset,
	processor_t                    processor,
	thread_t                       thread,
	sched_options_t               *options);
#endif /* !CONFIG_SCHED_SMT */

extern bool sched_SMT_balance(
	processor_t processor,
	processor_set_t pset);

extern void thread_quantum_init(
	thread_t thread,
	uint64_t now);


extern void             run_queue_init(
	run_queue_t             runq);

extern thread_t run_queue_dequeue(
	run_queue_t           runq,
	sched_options_t       options);

extern boolean_t        run_queue_enqueue(
	run_queue_t           runq,
	thread_t              thread,
	sched_options_t       options);

extern void     run_queue_remove(
	run_queue_t            runq,
	thread_t                       thread);

extern thread_t run_queue_peek(
	run_queue_t            runq);

struct sched_update_scan_context {
	uint64_t        earliest_bg_make_runnable_time;
	uint64_t        earliest_normal_make_runnable_time;
	uint64_t        earliest_rt_make_runnable_time;
	uint64_t        sched_tick_last_abstime;
};
typedef struct sched_update_scan_context *sched_update_scan_context_t;


extern void sched_pset_made_schedulable(
	processor_t processor,
	processor_set_t pset,
	boolean_t drop_lock);

extern void sched_cpu_init_completed(void);

/*
 * Enum to define various events which need IPIs. The IPI policy
 * engine decides what kind of IPI to use based on destination
 * processor state, thread and one of the following scheduling events.
 */
typedef enum {
	SCHED_IPI_EVENT_BOUND_THR   = 0x1,
	SCHED_IPI_EVENT_PREEMPT     = 0x2,
	SCHED_IPI_EVENT_SMT_REBAL   = 0x3,
	SCHED_IPI_EVENT_SPILL       = 0x4,
	SCHED_IPI_EVENT_REBALANCE   = 0x5,
	SCHED_IPI_EVENT_RT_PREEMPT  = 0x6,
} sched_ipi_event_t;


/* Enum to define various IPI types used by the scheduler */
typedef enum {
	SCHED_IPI_NONE              = 0x0,
	SCHED_IPI_IMMEDIATE         = 0x1,
	SCHED_IPI_IDLE              = 0x2,
	SCHED_IPI_DEFERRED          = 0x3,
} sched_ipi_type_t;

/* The IPI policy engine behaves in the following manner:
 * - All scheduler events which need an IPI invoke sched_ipi_action() with
 *   the appropriate destination processor, thread and event.
 * - sched_ipi_action() performs basic checks, invokes the scheduler specific
 *   ipi_policy routine and sets pending_AST bits based on the result.
 * - Once the pset lock is dropped, the scheduler invokes sched_ipi_perform()
 *   routine which actually sends the appropriate IPI to the destination core.
 */
extern sched_ipi_type_t sched_ipi_action(processor_t dst, thread_t thread, sched_ipi_event_t event);
extern void sched_ipi_perform(processor_t dst, sched_ipi_type_t ipi);

/* sched_ipi_policy() is the global default IPI policy for all schedulers */
extern sched_ipi_type_t sched_ipi_policy(processor_t dst, thread_t thread,
    boolean_t dst_idle, sched_ipi_event_t event);

/* sched_ipi_deferred_policy() is the global default deferred IPI policy for all schedulers */
extern sched_ipi_type_t sched_ipi_deferred_policy(processor_set_t pset,
    processor_t dst, thread_t thread, sched_ipi_event_t event);

#if defined(CONFIG_SCHED_TIMESHARE_CORE)

extern boolean_t        thread_update_add_thread(thread_t thread);
extern void             thread_update_process_threads(void);
extern boolean_t        runq_scan(run_queue_t runq, sched_update_scan_context_t scan_context);

#if CONFIG_SCHED_CLUTCH
extern boolean_t        sched_clutch_timeshare_scan(queue_t thread_queue, uint16_t count, sched_update_scan_context_t scan_context);
#endif /* CONFIG_SCHED_CLUTCH */

extern void sched_timeshare_init(void);
extern void sched_timeshare_timebase_init(void);
extern void sched_timeshare_maintenance_continue(void);



extern boolean_t priority_is_urgent(int priority);
extern uint32_t sched_timeshare_initial_quantum_size(thread_t thread);

extern int sched_compute_timeshare_priority(thread_t thread);

#endif /* CONFIG_SCHED_TIMESHARE_CORE */

/* Remove thread from its run queue */
extern boolean_t        thread_run_queue_remove(thread_t thread);
thread_t thread_run_queue_remove_for_handoff(thread_t thread);

/* Put a thread back in the run queue after being yanked */
extern void thread_run_queue_reinsert(thread_t thread, sched_options_t options);

extern void             thread_timer_expire(
	void                    *thread,
	void                    *p1);

extern bool thread_is_eager_preempt(thread_t thread);

extern boolean_t sched_generic_direct_dispatch_to_idle_processors;

/* Set the maximum interrupt level for the thread */
__private_extern__ wait_interrupt_t thread_interrupt_level(
	wait_interrupt_t interruptible);

__private_extern__ wait_result_t thread_mark_wait_locked(
	thread_t                 thread,
	wait_interrupt_t interruptible);

/* Wake up locked thread directly, passing result */
__private_extern__ kern_return_t clear_wait_internal(
	thread_t                thread,
	wait_result_t   result);

struct sched_statistics {
	uint32_t        csw_count;
	uint32_t        preempt_count;
	uint32_t        preempted_rt_count;
	uint32_t        preempted_by_rt_count;
	uint32_t        rt_sched_count;
	uint32_t        interrupt_count;
	uint32_t        ipi_count;
	uint32_t        timer_pop_count;
	uint32_t        idle_transitions;
	uint32_t        quantum_timer_expirations;
};
PERCPU_DECL(struct sched_statistics, sched_stats);
extern bool             sched_stats_active;

extern void sched_stats_handle_csw(
	processor_t processor,
	int reasons,
	int selfpri,
	int otherpri);

extern void sched_stats_handle_runq_change(
	struct runq_stats *stats,
	int old_count);

#define SCHED_STATS_INC(field)                                                  \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_stats_active)) {                                 \
	        PERCPU_GET(sched_stats)->field++;                               \
	}                                                                       \
MACRO_END

#if DEBUG

#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri)                  \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_stats_active)) {                                 \
	        sched_stats_handle_csw((processor),                             \
	            (reasons), (selfpri), (otherpri));                          \
	}                                                                       \
MACRO_END


#define SCHED_STATS_RUNQ_CHANGE(stats, old_count)                               \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_stats_active)) {                                 \
	        sched_stats_handle_runq_change((stats), (old_count));           \
	}                                                                       \
MACRO_END

#else /* DEBUG */

#define SCHED_STATS_CSW(processor, reasons, selfpri, otherpri) do { }while(0)
#define SCHED_STATS_RUNQ_CHANGE(stats, old_count) do { }while(0)

#endif /* DEBUG */

extern uint32_t sched_debug_flags;
#define SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS           0x00000001
#define SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS   0x00000002
#define SCHED_DEBUG_FLAG_AST_CHECK_TRACEPOINTS   0x00000004

#define SCHED_DEBUG_PLATFORM_KERNEL_DEBUG_CONSTANT(...)                         \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_debug_flags &                                    \
	    SCHED_DEBUG_FLAG_PLATFORM_TRACEPOINTS)) {                           \
	        KERNEL_DEBUG_CONSTANT(__VA_ARGS__);                             \
	}                                                                       \
MACRO_END

#define SCHED_DEBUG_CHOOSE_PROCESSOR_KERNEL_DEBUG_CONSTANT_IST(...)             \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_debug_flags &                                    \
	    SCHED_DEBUG_FLAG_CHOOSE_PROCESSOR_TRACEPOINTS)) {                   \
	        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, __VA_ARGS__);           \
	}                                                                       \
MACRO_END

#define SCHED_DEBUG_AST_CHECK_KDBG_RELEASE(...)                                 \
MACRO_BEGIN                                                                     \
	if (__improbable(sched_debug_flags &                                    \
	    SCHED_DEBUG_FLAG_AST_CHECK_TRACEPOINTS)) {                          \
	        KDBG_RELEASE(__VA_ARGS__);                                      \
	}                                                                       \
MACRO_END


/* Tells if there are "active" RT threads in the system (provided by CPU PM) */
extern void     active_rt_threads(
	boolean_t       active);

/* Returns the perfcontrol attribute for the thread */
extern perfcontrol_class_t thread_get_perfcontrol_class(
	thread_t        thread);

/* Generic routine for Non-AMP schedulers to calculate parallelism */
extern uint32_t sched_qos_max_parallelism(int qos, uint64_t options);

extern void check_monotonic_time(uint64_t ctime);

#endif /* defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS */

__BEGIN_DECLS

#ifdef  XNU_KERNEL_PRIVATE

extern kern_return_t thread_soft_bind_cluster_type(thread_t, char cluster_type);

__options_decl(thread_bind_option_t, uint64_t, {
	/* Unbind a previously cluster bound thread */
	THREAD_UNBIND                   = 0x1,
	/*
	 * Bind thread to the cluster only if it is eligible to run on that cluster. If
	 * the thread is not eligible to run on the cluster, thread_soft_bind_cluster_id()
	 * returns KERN_INVALID_POLICY.
	 */
	THREAD_BIND_ELIGIBLE_ONLY       = 0x2,
});
extern kern_return_t thread_soft_bind_cluster_id(thread_t thread, uint32_t cluster_id, thread_bind_option_t options);

extern int sched_get_rt_n_backup_processors(void);
extern void sched_set_rt_n_backup_processors(int n);

extern int sched_get_rt_deadline_epsilon(void);
extern void sched_set_rt_deadline_epsilon(int new_epsilon_us);

/* Toggles a global override to turn off CPU Throttling */
extern void     sys_override_cpu_throttle(boolean_t enable_override);

extern int sched_get_powered_cores(void);
extern void sched_set_powered_cores(int n);

uint64_t sched_sysctl_get_recommended_cores(void);

/*
 ****************** Only exported until BSD stops using ********************
 */

extern void                     thread_vm_bind_group_add(void);

/* Wake up thread directly, passing result */
extern kern_return_t clear_wait(
	thread_t                thread,
	wait_result_t   result);

/* Start thread running */
extern void             thread_bootstrap_return(void) __attribute__((noreturn));

/* Return from exception (BSD-visible interface) */
extern void             thread_exception_return(void) __dead2;

#define SCHED_STRING_MAX_LENGTH (48)
/* String declaring the name of the current scheduler */
extern char sched_string[SCHED_STRING_MAX_LENGTH];

__options_decl(thread_handoff_option_t, uint32_t, {
	THREAD_HANDOFF_NONE          = 0,
	THREAD_HANDOFF_SETRUN_NEEDED = 0x1,
});

/* Remove thread from its run queue */
thread_t thread_prepare_for_handoff(thread_t thread, thread_handoff_option_t option);

/* Attempt to context switch to a specific runnable thread */
extern wait_result_t thread_handoff_deallocate(thread_t thread, thread_handoff_option_t option);

__attribute__((nonnull(2)))
extern void thread_handoff_parameter(thread_t thread,
    thread_continue_t continuation, void *parameter, thread_handoff_option_t) __dead2;

extern struct waitq     *assert_wait_queue(event_t event);

/*
 * sched_cond_t:
 *
 * A atomic condition variable used to synchronize wake/block operations on threads.
 * Bits defined below are reserved for use by sched_prim. Remaining
 * bits may be used by caller for additional synchronization semantics.
 */
__options_decl(sched_cond_t, uint32_t, {
	SCHED_COND_INIT = 0x0000,    /* initialize all bits to zero (inactive and not awoken) */
	SCHED_COND_ACTIVE = 0x0001,  /* target thread is active */
	SCHED_COND_WAKEUP = 0x0002   /* wakeup has been issued for target thread */
});
typedef _Atomic sched_cond_t sched_cond_atomic_t;

/*
 * sched_cond_init:
 *
 * Initialize an atomic condition variable. Note that this does not occur atomically and should be
 * performed during thread initialization, before the condition is observable by other threads.
 */
extern void sched_cond_init(
	sched_cond_atomic_t *cond);

/*
 * sched_cond_signal:
 *
 * Wakeup the specified thread if it is waiting on this event and it has not already been issued a wakeup.
 *
 * parameters:
 *      thread    thread to awaken
 *      cond      atomic condition variable
 */
extern kern_return_t sched_cond_signal(
	sched_cond_atomic_t *cond,
	thread_t thread);

/*
 * sched_cond_wait_parameter:
 *
 * Assert wait and block on cond if no wakeup has been issued.
 * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and
 * return `THREAD_AWAKENED`.
 *
 * `sched_cond_wait_parameter` must be paired with `sched_cond_ack`.
 *
 * NOTE: `continuation` will only be jumped to if a wakeup has not been issued
 *
 * parameters:
 *      cond             atomic condition variable to synchronize on
 *      interruptible    interruptible value to pass to assert_wait
 *      continuation     continuation if block succeeds
 *      parameter
 */
extern wait_result_t sched_cond_wait_parameter(
	sched_cond_atomic_t *cond,
	wait_interrupt_t interruptible,
	thread_continue_t continuation,
	void *parameter);

/*
 * sched_cond_wait:
 *
 * Assert wait and block on cond if no wakeup has been issued.
 * If a wakeup has been issued on cond since the last `sched_cond_ack`, clear_wait and
 * return `THREAD_AWAKENED`.
 *
 * `sched_cond_wait` must be paired with `sched_cond_ack`.
 *
 * NOTE: `continuation` will only be jumped to if a wakeup has not been issued
 *
 * parameters:
 *      cond             atomic condition variable to synchronize on
 *      interruptible    interruptible value to pass to assert_wait
 *      continuation     continuation if block succeeds
 */
extern wait_result_t sched_cond_wait(
	sched_cond_atomic_t *cond,
	wait_interrupt_t interruptible,
	thread_continue_t continuation);

/*
 * sched_cond_ack:
 *
 * Acknowledge an issued wakeup by clearing WAKEUP and setting ACTIVE (via XOR).
 * It is the callers responsibility to ensure that the ACTIVE bit is always low prior to calling
 * (i.e. by calling `sched_cond_wait` prior to any rerun or block).
 * Synchronization schemes that allow for WAKEUP bit to be reset prior to wakeup
 * (e.g. a cancellation mechanism) should check that WAKEUP was indeed cleared.
 *
 * e.g.
 * ```
 * if (sched_cond_ack(&my_state) & SCHED_THREAD_WAKEUP) {
 *     // WAKEUP bit was no longer set by the time this thread woke up
 *     do_cancellation_policy();
 * }
 * ```
 *
 * parameters:
 *      cond:    atomic condition variable
 */
extern sched_cond_t sched_cond_ack(
	sched_cond_atomic_t *cond);

#endif  /* XNU_KERNEL_PRIVATE */

#if defined(KERNEL_PRIVATE) || SCHED_TEST_HARNESS
/* Set pending block hint for a particular object before we go into a wait state */
extern void             thread_set_pending_block_hint(
	thread_t                        thread,
	block_hint_t                    block_hint);

#define QOS_PARALLELISM_COUNT_LOGICAL   0x1
#define QOS_PARALLELISM_REALTIME        0x2
#define QOS_PARALLELISM_CLUSTER_SHARED_RESOURCE              0x4

extern uint32_t qos_max_parallelism(int qos, uint64_t options);
#endif /* defined(KERNEL_PRIVATE) || SCHED_TEST_HARNESS */

#if XNU_KERNEL_PRIVATE
extern void             thread_yield_with_continuation(
	thread_continue_t       continuation,
	void                            *parameter) __dead2;
#endif /* XNU_KERNEL_PRIVATE */

/* Context switch */
extern wait_result_t    thread_block(
	thread_continue_t       continuation);

extern wait_result_t    thread_block_parameter(
	thread_continue_t       continuation,
	void                            *parameter);

/* Declare thread will wait on a particular event */
extern wait_result_t    assert_wait(
	event_t                         event,
	wait_interrupt_t        interruptible);

/* Assert that the thread intends to wait with a timeout */
extern wait_result_t    assert_wait_timeout(
	event_t                         event,
	wait_interrupt_t        interruptible,
	uint32_t                        interval,
	uint32_t                        scale_factor);

/* Assert that the thread intends to wait with an urgency, timeout and leeway */
extern wait_result_t    assert_wait_timeout_with_leeway(
	event_t                         event,
	wait_interrupt_t        interruptible,
	wait_timeout_urgency_t  urgency,
	uint32_t                        interval,
	uint32_t                        leeway,
	uint32_t                        scale_factor);

extern wait_result_t    assert_wait_deadline(
	event_t                         event,
	wait_interrupt_t        interruptible,
	uint64_t                        deadline);

/* Assert that the thread intends to wait with an urgency, deadline, and leeway */
extern wait_result_t    assert_wait_deadline_with_leeway(
	event_t                         event,
	wait_interrupt_t        interruptible,
	wait_timeout_urgency_t  urgency,
	uint64_t                        deadline,
	uint64_t                        leeway);


/* Wake up thread (or threads) waiting on a particular event */
extern kern_return_t    thread_wakeup_prim(
	event_t                         event,
	boolean_t                       one_thread,
	wait_result_t                   result);

/* Wake up up to given number of threads waiting on a particular event */
extern kern_return_t    thread_wakeup_nthreads_prim(
	event_t                         event,
	uint32_t                        nthreads,
	wait_result_t                   result);

#define thread_wakeup(x) \
	thread_wakeup_prim((x), FALSE, THREAD_AWAKENED)
#define thread_wakeup_with_result(x, z) \
	thread_wakeup_prim((x), FALSE, (z))
#define thread_wakeup_one(x) \
	thread_wakeup_prim((x), TRUE, THREAD_AWAKENED)

#define thread_wakeup_nthreads(x, nthreads) \
	thread_wakeup_nthreads_prim((x), (nthreads), THREAD_AWAKENED)
#define thread_wakeup_nthreads_with_result(x, nthreads, z) \
	thread_wakeup_nthreads_prim((x), (nthreads), (z))

/* Wakeup the specified thread if it is waiting on this event */
extern kern_return_t thread_wakeup_thread(event_t event, thread_t thread);

extern boolean_t preemption_enabled(void);

#if defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS


#if   !CONFIG_SCHED_TIMESHARE_CORE && !CONFIG_SCHED_CLUTCH && !CONFIG_SCHED_EDGE
#error Enable at least one scheduler algorithm in osfmk/conf/MASTER.XXX
#endif

/*
 * The scheduling policy is fixed at compile-time, in order to save the performance
 * cost of function pointer indirection that we would otherwise pay each time when
 * making a policy-specific callout.
 */
#if __AMP__

#if CONFIG_SCHED_EDGE
extern const struct sched_dispatch_table sched_edge_dispatch;
#define SCHED(f) (sched_edge_dispatch.f)
#else /* CONFIG_SCHED_EDGE */
extern const struct sched_dispatch_table sched_amp_dispatch;
#define SCHED(f) (sched_amp_dispatch.f)
#endif /* CONFIG_SCHED_EDGE */

#else /* __AMP__ */

#if CONFIG_SCHED_CLUTCH
extern const struct sched_dispatch_table sched_clutch_dispatch;
#define SCHED(f) (sched_clutch_dispatch.f)
#else /* CONFIG_SCHED_CLUTCH */
extern const struct sched_dispatch_table sched_dualq_dispatch;
#define SCHED(f) (sched_dualq_dispatch.f)
#endif /* CONFIG_SCHED_CLUTCH */

#endif /* __AMP__ */

struct sched_dispatch_table {
	const char *sched_name;
	void    (*init)(void);                          /* Init global state */
	void    (*timebase_init)(void);         /* Timebase-dependent initialization */
	void    (*processor_init)(processor_t processor);       /* Per-processor scheduler init */
	void    (*pset_init)(processor_set_t pset);     /* Per-processor set scheduler init */

	void    (*maintenance_continuation)(void);      /* Function called regularly */

	/*
	 * Choose a thread of greater or equal priority from the per-processor
	 * runqueue for timeshare/fixed threads
	 */
	thread_t        (*choose_thread)(
		processor_t           processor,
		int                           priority,
		thread_t              prev_thread,
		ast_t reason);

	/* True if scheduler supports stealing threads for this pset */
	bool    (*steal_thread_enabled)(processor_set_t pset);

	/*
	 * Steal a thread from another processor in the pset so that it can run
	 * immediately
	 */
	thread_t        (*steal_thread)(
		processor_set_t         pset);

	/*
	 * Compute priority for a timeshare thread based on base priority.
	 */
	int (*compute_timeshare_priority)(thread_t thread);

	/*
	 * Pick the best node for a thread to run on.
	 */
	pset_node_t (*choose_node)(
		thread_t                      thread);

	/*
	 * Pick the best processor for a thread (any kind of thread) to run on.
	 */
	processor_t     (*choose_processor)(
		processor_set_t                pset,
		processor_t                    processor,
		thread_t                       thread,
		sched_options_t               *options);

	/*
	 * Enqueue a timeshare or fixed priority thread onto the per-processor
	 * runqueue
	 */
	boolean_t (*processor_enqueue)(
		processor_t                    processor,
		thread_t                       thread,
		sched_options_t                options);

	/* Migrate threads away in preparation for processor shutdown */
	void (*processor_queue_shutdown)(
		processor_t                    processor);

	/* Remove the specific thread from the per-processor runqueue */
	boolean_t       (*processor_queue_remove)(
		processor_t             processor,
		thread_t                thread);

	/*
	 * Does the per-processor runqueue have any timeshare or fixed priority
	 * threads on it? Called without pset lock held, so should
	 * not assume immutability while executing.
	 */
	boolean_t       (*processor_queue_empty)(processor_t            processor);

	/*
	 * Would this priority trigger an urgent preemption if it's sitting
	 * on the per-processor runqueue?
	 */
	boolean_t       (*priority_is_urgent)(int priority);

	/*
	 * Does the per-processor runqueue contain runnable threads that
	 * should cause the currently-running thread to be preempted?
	 */
	ast_t           (*processor_csw_check)(processor_t processor);

	/*
	 * Does the per-processor runqueue contain a runnable thread
	 * of > or >= priority, as a preflight for choose_thread() or other
	 * thread selection
	 */
	boolean_t       (*processor_queue_has_priority)(processor_t             processor,
	    int                             priority,
	    boolean_t               gte);

	/* Quantum size for the specified non-realtime thread. */
	uint32_t        (*initial_quantum_size)(thread_t thread);

	/* Scheduler mode for a new thread */
	sched_mode_t    (*initial_thread_sched_mode)(task_t parent_task);

	/*
	 * Is it safe to call update_priority, which may change a thread's
	 * runqueue or other state. This can be used to throttle changes
	 * to dynamic priority.
	 */
	boolean_t       (*can_update_priority)(thread_t thread);

	/*
	 * Update both scheduled priority and other persistent state.
	 * Side effects may including migration to another processor's runqueue.
	 */
	void            (*update_priority)(thread_t thread);

	/* Lower overhead update to scheduled priority and state. */
	void            (*lightweight_update_priority)(thread_t thread);

	/* Callback for non-realtime threads when the quantum timer fires */
	void            (*quantum_expire)(thread_t thread);

	/*
	 * Runnable threads on per-processor runqueue. Should only
	 * be used for relative comparisons of load between processors.
	 */
	int                     (*processor_runq_count)(processor_t     processor);

	/* Aggregate runcount statistics for per-processor runqueue */
	uint64_t    (*processor_runq_stats_count_sum)(processor_t   processor);

	boolean_t       (*processor_bound_count)(processor_t processor);

	void            (*thread_update_scan)(sched_update_scan_context_t scan_context);

	/* Supports more than one pset */
	boolean_t   multiple_psets_enabled;

	/* Supports avoid-processor */
	boolean_t   avoid_processor_enabled;

	/* Returns true if this processor should avoid running this thread. */
	bool    (*thread_avoid_processor)(processor_t processor, thread_t thread, ast_t reason);

	/*
	 * Invoked when a processor is about to choose the idle thread
	 * Used to send IPIs to a processor which would be preferred to be idle instead.
	 * Returns true if the current processor should anticipate a quick IPI reply back
	 * from another core.
	 * Called with pset lock held, returns with pset lock unlocked.
	 */
	bool    (*processor_balance)(processor_t processor, processor_set_t pset);

	processor_t (*rt_choose_processor)(processor_set_t starting_pset, processor_t starting_processor, thread_t thread);
	thread_t    (*rt_steal_thread)(processor_set_t stealing_pset);
	void    (*rt_init_pset)(processor_set_t pset);
	void    (*rt_init_completed)(void);
	void    (*rt_queue_shutdown)(processor_t processor);
	void    (*rt_runq_scan)(sched_update_scan_context_t scan_context);
	int64_t (*rt_runq_count_sum)(void);

	uint32_t (*qos_max_parallelism)(int qos, uint64_t options);
	void    (*check_spill)(processor_set_t pset, thread_t thread);
	sched_ipi_type_t (*ipi_policy)(processor_t dst, thread_t thread, boolean_t dst_idle, sched_ipi_event_t event);
	bool    (*thread_should_yield)(processor_t processor, thread_t thread);

	/* Routine to update run counts */
	uint32_t (*run_count_incr)(thread_t thread);
	uint32_t (*run_count_decr)(thread_t thread);

	/* Routine to update scheduling bucket for a thread */
	void (*update_thread_bucket)(thread_t thread);

	/* Routine to inform the scheduler when a new pset becomes schedulable */
	void (*pset_made_schedulable)(processor_t processor, processor_set_t pset, boolean_t drop_lock);
#if CONFIG_THREAD_GROUPS
	/* Routine to inform the scheduler when CLPC changes a thread group recommendation */
	void (*thread_group_recommendation_change)(struct thread_group *tg, cluster_type_t new_recommendation);
#endif
	/* Routine to inform the scheduler when all CPUs have finished initializing */
	void (*cpu_init_completed)(void);
	/* Routine to check if a thread is eligible to execute on a specific pset */
	bool (*thread_eligible_for_pset)(thread_t thread, processor_set_t pset);
};

extern const struct sched_dispatch_table sched_dualq_dispatch;
#if __AMP__
extern const struct sched_dispatch_table sched_amp_dispatch;
#endif

#if defined(CONFIG_SCHED_CLUTCH)
extern const struct sched_dispatch_table sched_clutch_dispatch;
#endif

#if defined(CONFIG_SCHED_EDGE)
extern const struct sched_dispatch_table sched_edge_dispatch;
#endif

extern void sched_set_max_unsafe_rt_quanta(int max);
extern void sched_set_max_unsafe_fixed_quanta(int max);

#endif  /* defined(MACH_KERNEL_PRIVATE) || SCHED_TEST_HARNESS */

__END_DECLS

#endif  /* _KERN_SCHED_PRIM_H_ */
