/*
* Copyright (c) 2000-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
#ifdef XNU_KERNEL_PRIVATE
#ifndef _KERN_STARTUP_H_
#define _KERN_STARTUP_H_
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <libkern/section_keywords.h>
__BEGIN_DECLS
#pragma GCC visibility push(hidden)
/*!
* @enum startup_subsystem_id_t
*
* @abstract
* Represents a stage of kernel intialization, ubnd allows for subsystems
* to register initializers for a specific stage.
*
* @discussion
* Documentation of each subsystem initialization sequence exists in
* @file doc/startup.md.
*/
__enum_decl(startup_subsystem_id_t, uint32_t, {
STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */
STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */
STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */
STARTUP_SUB_LOCKS, /**< various subsystem locks */
STARTUP_SUB_KPRINTF, /**< kprintf initialization */
STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */
STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */
STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */
STARTUP_SUB_KTRACE, /**< initialize kernel trace */
STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */
STARTUP_SUB_EVENT, /**< initiailze the event subsystem */
STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */
STARTUP_SUB_OSLOG, /**< oslog and kernel logging */
STARTUP_SUB_MACH_IPC, /**< Mach IPC */
STARTUP_SUB_THREAD_CALL, /**< Thread calls */
STARTUP_SUB_SYSCTL, /**< registers sysctls */
STARTUP_SUB_EXCLAVES, /**< initialize exclaves */
STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */
STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */
});
/*!
* Stores the last subsystem to have been fully initialized;
*/
extern startup_subsystem_id_t startup_phase;
/*!
* @enum startup_debug_t
*
* @abstract
* Flags set in the @c startup_debug global to configure startup debugging.
*/
__options_decl(startup_debug_t, uint32_t, {
STARTUP_DEBUG_NONE = 0x00000000,
STARTUP_DEBUG_VERBOSE = 0x00000001,
});
extern startup_debug_t startup_debug;
/*!
* @enum startup_rank
*
* @abstract
* Specifies in which rank a given initializer runs within a given section
* to register initializers for a specific rank within the subsystem.
*
* @description
* A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify
* a rank within the subsystem they initialize.
*
* @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based).
*
* @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and
* @c STARTUP_RANK_FOURTH are given as convenient names for these.
*
* @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions
* run after all the @c STARTUP_RANK_NTH(n) ones have.
*
* @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank
* after the @c STARTUP_RANK_MIDDLE ones (0-based).
*
* @c STARTUP_RANK_LAST callbacks will run absolutely last after everything
* else did for this subsystem.
*/
__enum_decl(startup_rank_t, uint32_t, {
#define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1))
STARTUP_RANK_FIRST = 0,
STARTUP_RANK_SECOND = 1,
STARTUP_RANK_THIRD = 2,
STARTUP_RANK_FOURTH = 3,
STARTUP_RANK_MIDDLE = 0x7fffffff,
#define STARTUP_RANK_LATE_NTH(n) \
((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n)))
STARTUP_RANK_LAST = 0xffffffff,
});
#if KASAN
/*
* The use of weird sections that get unmapped confuse the hell out of kasan,
* so for KASAN leave things in regular __TEXT/__DATA segments
*/
#define STARTUP_CODE_SEGSECT "__TEXT,__text"
#define STARTUP_CONST_SEGSECT "__DATA_CONST,__init"
#define STARTUP_DATA_SEGSECT "__DATA,__init"
#define STARTUP_HOOK_SEGMENT "__DATA"
#define STARTUP_HOOK_SECTION "__init_entry_set"
#elif defined(__x86_64__)
/* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */
#define STARTUP_CODE_SEGSECT "__TEXT,__text"
#define STARTUP_CONST_SEGSECT "__KLDDATA,__const"
#define STARTUP_DATA_SEGSECT "__KLDDATA,__init"
#define STARTUP_HOOK_SEGMENT "__KLDDATA"
#define STARTUP_HOOK_SECTION "__init_entry_set"
#else
/* arm protects __KLD early, so use __BOOTDATA for data */
#define STARTUP_CODE_SEGSECT "__TEXT,__text"
#define STARTUP_CONST_SEGSECT "__KLDDATA,__const"
#define STARTUP_DATA_SEGSECT "__BOOTDATA,__init"
#define STARTUP_HOOK_SEGMENT "__BOOTDATA"
#define STARTUP_HOOK_SECTION "__init_entry_set"
#endif
/*!
* @macro __startup_func
*
* @abstract
* Attribute to place on functions used only during the kernel startup phase.
*
* @description
* Code marked with this attribute will be unmapped after kernel lockdown.
*/
#define __startup_func \
__PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \
__attribute__((cold, visibility("hidden")))
/*!
* @macro __startup_data
*
* @abstract
* Attribute to place on globals used during the kernel startup phase.
*
* @description
* Data marked with this attribute will be unmapped after kernel lockdown.
*/
#define __startup_data \
__PLACE_IN_SECTION(STARTUP_DATA_SEGSECT)
/*!
* @macro __startup_const
*
* @abstract
* Attribute to place on global constants used during the kernel startup phase.
*
* @description
* Data marked with this attribute will be unmapped after kernel lockdown.
*
* __startup_const implies const. Be mindful that for pointers, the `const`
* will end up on the wrong side of the star if you put __startup_const at the
* start of the declaration.
*/
#define __startup_const \
__PLACE_IN_SECTION(STARTUP_CONST_SEGSECT) const
/*!
* @macro STARTUP
*
* @abstract
* Declares a kernel startup callback.
*/
#define STARTUP(subsystem, rank, func) \
__STARTUP(func, __LINE__, subsystem, rank, func)
/*!
* @macro STARTUP_ARG
*
* @abstract
* Declares a kernel startup callback that takes an argument.
*/
#define STARTUP_ARG(subsystem, rank, func, arg) \
__STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg)
/*!
* @macro TUNABLE
*
* @abstract
* Declares a read-only kernel tunable that is read from a boot-arg with
* a default value, without further processing.
*
* @param type_t
* Should be an integer type or bool.
*
* @param var
* The name of the C variable to use for storage.
*
* @param boot_arg
* The name of the boot-arg to parse for initialization
*
* @param default_value
* The default value for the tunable if the boot-arg is absent.
*/
#define TUNABLE(type_t, var, boot_arg, default_value) \
SECURITY_READ_ONLY_LATE(type_t) var = default_value; \
__TUNABLE(type_t, var, boot_arg)
/*!
* @macro TUNABLE_WRITEABLE
*
* @abstract
* Declares a writeable kernel tunable that is read from a boot-arg with
* a default value, without further processing.
*
* @param type_t
* Should be an integer type or bool.
*
* @param var
* The name of the C variable to use for storage.
*
* @param boot_arg
* The name of the boot-arg to parse for initialization
*
* @param default_value
* The default value for the tunable if the boot-arg is absent.
*/
#define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \
type_t var = default_value; \
__TUNABLE(type_t, var, boot_arg)
#if DEBUG || DEVELOPMENT
#define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \
TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value)
#else
#define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \
TUNABLE(type_t, var, boot_arg, default_value)
#endif
/*!
* @macro TUNABLE_STR
*
* @abstract
* Declares a read-only kernel tunable that is read from a boot-arg with
* a default value, without further processing.
*
* @param var
* The name of the C variable to use for storage.
*
* @param count
* The number of bytes in the buffer.
*
* @param boot_arg
* The name of the boot-arg to parse for initialization
*
* @param default_value
* The default value for the tunable if the boot-arg is absent.
*/
#define TUNABLE_STR(var, count, boot_arg, default_value) \
char __security_const_late var[count] = default_value; \
__TUNABLE_STR(var, boot_arg)
/*!
* @enum tunable_dt_flags_t
*
* @abstract
* Flags used with the @c TUNABLE_DT* macros.
*
* @description
* If TUNABLE_DT_CHECK_CHOSEN is set, a value in
* /chosen/<dt_base>/<dt_name> takes precedence over any value in
* /<dt_base>/<dt_name>. /chosen is by convention the area where
* synthesized values not coming from the serialized device tree are
* being added, so this provides a way for e.g. the boot-loader to
* set/override tunables.
*/
__options_decl(tunable_dt_flags_t, uint32_t, {
TUNABLE_DT_NONE = 0x00000000,
TUNABLE_DT_CHECK_CHOSEN = 0x00000001,
});
/*!
* @macro TUNABLE_DT
*
* @abstract
* Like TUNABLE, but gets the initial value from both Device Tree and
* boot-args. The order in which the initial value is resolved is as
* follows, with later steps overriding previous ones (if they are
* specified):
*
* 1. Device Tree Entry "/<dt_base>/<dt_name>",
* 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry
* "/chosen/<dt_base>/<dt_name>" (see the description for
* @c tunable_dt_flags_t),
* 3. boot-args.
*
* @param type_t
* Should be an integer type or bool.
*
* @param var
* The name of the C variable to use for storage.
*
* @param dt_base
* The name of the DT node containing the property.
*
* @param dt_name
* The name of the DT property containing the default value.
*
* @param boot_arg
* The name of the boot-arg overriding the initial value from the DT.
*
* @param default_value
* The default value for the tunable if both DT entry and boot-arg are
* absent.
*
* @param flags
* See the description for @c tunable_dt_flags_t.
*/
#define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
SECURITY_READ_ONLY_LATE(type_t) var = default_value; \
__TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags)
/*!
* @macro TUNABLE_DT_WRITEABLE
*
* @abstract
* Like TUNABLE_WRITEABLE, but gets the initial value from both Device
* Tree and boot-args. The order in which the initial value is
* resolved is as follows, with later steps overriding previous ones
* (if they are specified):
*
* 1. Device Tree Entry "/<dt_base>/<dt_name>",
* 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry
* "/chosen/<dt_base>/<dt_name>" (see the description for
* @c tunable_dt_flags_t),
* 3. boot-args.
*
* @param type_t
* Should be an integer type or bool.
*
* @param var
* The name of the C variable to use for storage.
*
* @param dt_base
* The name of the DT node containing the property.
*
* @param dt_name
* The name of the DT property containing the default value.
*
* @param boot_arg
* The name of the boot-arg overriding the initial value from the DT.
*
* @param default_value
* The default value for the tunable if both DT entry and boot-arg are
* absent.
*
* @param flags
* See the description for @c tunable_dt_flags_t.
*/
#define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
type_t var = default_value; \
__TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags)
#if DEBUG || DEVELOPMENT
#define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags)
#else
#define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \
TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags)
#endif
/*
* Machine Timeouts
*
* Machine Timeouts are timeouts for low level kernel code manifesting
* as _Atomic uint64_t variables, whose default value can be
* overridden and scaled via the device tree and boot-args.
*
* Each timeout has a name, looked up directly as the property name in
* the device tree in both the "/machine-timeouts" and
* "/chosen/machine-timeouts" nodes. The "chosen" property always
* overrides the other one. This allows fixed per-device timeouts in
* the device tree to be overridden by iBoot in "chosen".
*
* Additionally, the same name with "-scale" appended is looked up as
* properties for optional scale factors. Scale factors are not
* overridden by chosen, instead all scale factors (including global
* and/or boot-arg scale factors) combine by multiplication.
*
* The special name "global-scale" provides a scale that applies to
* every timeout.
*
* All property names can be used as boot-args by prefixing
* "ml-timeout-", e.g. th global scale is available as the
* "ml-timeout-global-scale" boot-arg.
*
* By convention, if the timeout value resolves to 0, the timeout
* should be disabled.
*/
/*
* Machine Timeouts types. See the next section for what unit
* they are in.
*
* We use _Atomic, but only with relaxed ordering: This is just to
* make sure all devices see consistent values all the time. Since
* the actual timeout value will be seen as 0 before initializaton,
* relaxed ordering means that code that runs concurrently with
* initialization only risks to see a disabled timeout during early
* boot.
*/
typedef _Atomic uint64_t machine_timeout_t;
/*
* Units
*
* Machine Timeouts are ALWAYS in picoseconds in the device tree or
* boot-args, to avoid confusion when changing or comparing timeouts
* as a user, but the actual storage value might contain the same
* duration in another unit, calculated by the initialization code.
*
* This is done because otherwise we would likely introduce another
* multiplication in potentially hot code paths, given that code that
* actually uses the timeout storage variable is unlikely to work with
* picosecond values when comparing against the timeout deadline.
*
* This unit scale is *only* applied during initialization at early
* boot, and only if the timeout's default value was overridden
* through the device tree or a boot-arg.
*/
#define MACHINE_TIMEOUT_UNIT_PSEC 1
#define MACHINE_TIMEOUT_UNIT_NSEC 1000
#define MACHINE_TIMEOUT_UNIT_USEC (1000*1000)
#define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000)
// Special unit for timebase ticks (usually 1/24MHz)
#define MACHINE_TIMEOUT_UNIT_TIMEBASE 0
// DT property names are limited to 31 chars, minus "-global" suffix
#define MACHINE_TIMEOUT_MAX_NAME_LEN 25
struct machine_timeout_spec {
void *ptr;
uint64_t default_value;
uint64_t unit_scale;
char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1];
bool (*skip_predicate)(struct machine_timeout_spec const *);
};
extern void
machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix);
extern void
machine_timeout_init(const struct machine_timeout_spec *spec);
#if DEVELOPMENT || DEBUG
// Late timeout (re-)initialization, at the end of bsd_init()
extern void
machine_timeout_bsd_init(void);
#endif /* DEVELOPMENT || DEBUG */
/*!
* @macro MACHINE_TIMEOUT and MACHINE_TIMEOUT_DEV_WRITEABLE
*
* @abstract
* Defines a Machine Timeout that can be overridden and
* scaled through the device tree and boot-args.
*
* The variant with the _DEV_WRITEABLE suffix does not mark the timeout as
* SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g.
* machine_timeout_init_with_suffix or sysctls can change it after lockdown.
*
* @param var
* The name of the C variable to use for storage. If the storage value
* contains 0, the timeout is considered disabled by convention.
*
* @param timeout_name
* The name of the timeout, used for property and boot-arg names. See
* the general description of Machine Timeouts above for how this name
* ends up being used.
*
* @param timeout_default
* The default value for the timeout if not specified through device
* tree or boot-arg. Will still be scaled if a scale factor exists.
*
* @param var_unit
* The unit that the storage variable is in. Note that timeout values
* must always be specified as picoseconds in the device tree and
* boot-args, but timeout initialization will convert the value to the
* unit specified here before writing it to the storage variable.
*
* @param skip_predicate
* Optionally, a function to call to decide whether the timeout should
* be set or not. If NULL, the timeout will always be set (if
* specified anywhere). A predicate has the following signature:
* bool skip_predicate (struct machine_timeout_spec const *)
*/
#define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred) \
struct machine_timeout_spec \
__machine_timeout_spec_ ## var = { \
.ptr = &var, \
.default_value = timeout_default, \
.unit_scale = var_unit, \
.name = timeout_name, \
.skip_predicate = skip_pred, \
}; \
__STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \
machine_timeout_init, &__machine_timeout_spec_ ## var)
#define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \
SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \
_MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
#if DEVELOPMENT || DEBUG
#define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \
machine_timeout_t var = 0; \
_MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
#else
#define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \
MACHINE_TIMEOUT(var, name, default, unit, skip_predicate)
#endif /* DEVELOPMENT || DEBUG */
/*!
* @macro MACHINE_TIMEOUT_SPEC_REF
*
* @abstract
* References a previously defined MACHINE_TIMEOUT.
*
* This is primarily useful for overriding individual timeouts
* at arbitrary times (even after boot), by manually calling
* machine_timeout_init_with_suffix() with this macro
* as first argument, and a suffix to apply to both device tree and
* boot-arg as second argument.
*
* @param var
* The name of the C variable used for storage, as it was specified
* in MACHINE_TIMEOUT.
*/
#define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var)
/*!
* @macro MACHINE_TIMEOUT_SPEC_DECL
*
* @abstract
* Declaration of machine timeout spec, mostly useful to make it known
* for MACHINE_TIMEOUT_SPEC_REF.
*
* @param var
* The name of the C variable used for storage, as it was specified
* in MACHINE_TIMEOUT.
*/
#define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var
/*
* Event subsystem
*
* This allows to define a few system-wide events that allow for loose coupling
* between subsystems interested in those events and the emitter.
*/
/*!
* @macro EVENT_DECLARE()
*
* @brief
* Declares an event namespace with a given callback type.
*
* @param name The name for the event (typically in all caps).
* @param cb_type_t A function type for the callbacks.
*/
#define EVENT_DECLARE(name, cb_type_t) \
struct name##_event { \
struct event_hdr evt_link; \
cb_type_t *evt_cb; \
}; \
extern struct event_hdr name##_HEAD
/*!
* @macro EVENT_DEFINE()
*
* @brief
* Defines the head for the event corresponding to an EVENT_DECLARE()
*/
#define EVENT_DEFINE(name) \
__security_const_late struct event_hdr name##_HEAD
/*!
* @macro EVENT_REGISTER_HANDLER()
*
* @brief
* Registers a handler for a given event.
*
* @param name The name for the event as declared in EVENT_DECLARE()
* @param handler The handler to register for this event.
*/
#define EVENT_REGISTER_HANDLER(name, handler) \
__EVENT_REGISTER(name, __LINE__, handler)
/*!
* @macro EVENT_INVOKE()
*
* @brief
* Call all the events handlers with the specified arguments.
*
* @param name The name for the event as declared in EVENT_DECLARE()
* @param handler The handler to register for this event.
*/
#define EVENT_INVOKE(name, ...) \
for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \
__container_of(__e, struct name##_event, \
evt_link)->evt_cb(__VA_ARGS__); \
}
#if DEBUG || DEVELOPMENT
/*!
* @macro SYSCTL_TEST_REGISTER
*
* @abstract
* Declares a test that will appear under @c debug.test.${name}.
*
* @param name
* An identifier that will be stringified to form the sysctl test name.
*
* @param cb
* The callback to run, of type:
* <code>
* int (callback *)(int64_t value, int64_t *);
* </code>
*/
#define SYSCTL_TEST_REGISTER(name, cb) \
static __startup_data struct sysctl_test_setup_spec \
__startup_SYSCTL_TEST_ ## name = { \
.st_name = #name, \
.st_func = &cb, \
}; \
STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \
sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name)
#endif /* DEBUG || DEVELOPMENT */
#pragma mark - internals
__END_DECLS
#ifdef __cplusplus
template <typename T>
struct __startup_tunable {
static const bool value = false;
};
template <>
struct __startup_tunable <bool>{
static const bool value = true;
};
#define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value
#else
#define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t)
#endif
__BEGIN_DECLS
#define __TUNABLE(type_t, var, key) \
static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \
static __startup_const struct startup_tunable_spec \
__startup_TUNABLES_spec_ ## var = { \
.name = __startup_TUNABLES_name_ ## var, \
.var_addr = (void *)&var, \
.var_len = sizeof(type_t), \
.var_is_bool = __startup_type_is_bool(type_t), \
}; \
__STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var)
#define __TUNABLE_STR(var, key) \
static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \
static __startup_const struct startup_tunable_spec \
__startup_TUNABLES_spec_ ## var = { \
.name = __startup_TUNABLES_name_ ## var, \
.var_addr = (void *)&var, \
.var_len = sizeof(var), \
.var_is_str = true, \
}; \
__STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var)
#define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \
static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \
static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \
static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \
static __startup_const struct startup_tunable_dt_spec \
__startup_TUNABLES_DT_spec_ ## var = { \
.dt_base = __startup_TUNABLES_dt_base_ ## var, \
.dt_name = __startup_TUNABLES_dt_name_ ## var, \
.dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \
.boot_arg_name = __startup_TUNABLES_name_ ## var, \
.var_addr = (void *)&var, \
.var_len = sizeof(type_t), \
.var_is_bool = __startup_type_is_bool(type_t), \
}; \
__STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \
kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var)
#ifdef __cplusplus
#define __STARTUP_FUNC_CAST(func, a) \
(void(*)(const void *))func
#else
#define __STARTUP_FUNC_CAST(func, a) \
(typeof(func(a))(*)(const void *))func
#endif
#define __STARTUP1(name, line, subsystem, rank, func, a, b) \
__PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \
static const struct startup_entry \
__startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \
STARTUP_SUB_ ## subsystem, \
rank, __STARTUP_FUNC_CAST(func, a), b, \
}
#define __STARTUP(name, line, subsystem, rank, func) \
__STARTUP1(name, line, subsystem, rank, func, , NULL)
#define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \
__STARTUP1(name, line, subsystem, rank, func, arg, arg)
struct startup_entry {
startup_subsystem_id_t subsystem;
startup_rank_t rank;
void (*func)(const void *);
const void *arg;
};
struct startup_tunable_spec {
const char *name;
void *var_addr;
int var_len;
bool var_is_bool;
bool var_is_str;
};
struct startup_tunable_dt_spec {
const char *dt_base;
const char *dt_name;
bool dt_chosen_override;
const char *boot_arg_name;
void *var_addr;
int var_len;
bool var_is_bool;
};
#if DEBUG || DEVELOPMENT
struct sysctl_test_setup_spec {
const char *st_name;
int (*st_func)(int64_t, int64_t *);
};
extern void sysctl_register_test_startup(
struct sysctl_test_setup_spec *spec);
#endif /* DEBUG || DEVELOPMENT */
/*
* Kernel and machine startup declarations
*/
/* Initialize kernel */
extern void kernel_startup_bootstrap(void);
extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto);
extern void kernel_startup_tunable_init(const struct startup_tunable_spec *);
extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *);
extern void kernel_bootstrap(void);
/* Initialize machine dependent stuff */
extern void machine_init(void);
extern void secondary_cpu_main(void *machine_param);
/* Secondary cpu initialization */
extern void machine_cpu_reinit(void *machine_param);
extern void processor_cpu_reinit(void* machine_param, bool wait_for_cpu_signal, bool is_final_system_sleep);
/* Device subsystem initialization */
extern void device_service_create(void);
struct event_hdr {
struct event_hdr *next;
};
extern void event_register_handler(struct event_hdr *event_hdr);
#define __EVENT_REGISTER(name, lno, handler) \
static __security_const_late struct name##_event name##_event_##lno = { \
.evt_link.next = &name##_HEAD, \
.evt_cb = (handler), \
}; \
__STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \
event_register_handler, &name##_event_##lno.evt_link)
#ifdef MACH_BSD
/* BSD subsystem initialization */
extern void bsd_init(void);
extern int serverperfmode;
#if defined(XNU_TARGET_OS_OSX)
static inline bool
kernel_is_macos_or_server(void)
{
return true;
}
#else /* XNU_TARGET_OS_OSX */
static inline bool
kernel_is_macos_or_server(void)
{
return !!serverperfmode;
}
#endif /* XNU_TARGET_OS_OSX */
#endif /* MACH_BSD */
#pragma GCC visibility pop
__END_DECLS
#endif /* _KERN_STARTUP_H_ */
#endif /* XNU_KERNEL_PRIVATE */