This is xnu-11215.1.10. See this file in:
/*
* Copyright (c) 2007-2021 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
/*
* @APPLE_FREE_COPYRIGHT@
*/
/*
* File: arm/rtclock.c
* Purpose: Routines for handling the machine dependent
* real-time clock.
*/
#include <mach/mach_types.h>
#include <kern/clock.h>
#include <kern/thread.h>
#include <kern/macro_help.h>
#include <kern/spl.h>
#include <kern/timer_queue.h>
#include <kern/host_notify.h>
#include <machine/commpage.h>
#include <machine/machine_routines.h>
#include <machine/config.h>
#include <arm/exception.h>
#include <arm/cpu_data_internal.h>
#if __arm64__
#include <arm64/proc_reg.h>
#else
#error Unsupported arch
#endif
#include <arm/rtclock.h>
#include <IOKit/IOPlatformExpert.h>
#include <libkern/OSAtomic.h>
#include <sys/kdebug.h>
#define MAX_TIMEBASE_TRIES 10
int rtclock_init(void);
static int
deadline_to_decrementer(uint64_t deadline,
uint64_t now);
static void
timebase_callback(struct timebase_freq_t * freq);
#if DEVELOPMENT || DEBUG
uint32_t timebase_validation = 0;
#endif
/*
* Configure the real-time clock device at boot
*/
void
rtclock_early_init(void)
{
PE_register_timebase_callback(timebase_callback);
#if DEVELOPMENT || DEBUG
uint32_t tmp_mv = 1;
#if defined(APPLE_ARM64_ARCH_FAMILY)
/* Enable MAT validation on A0 hardware by default. */
timebase_validation = ml_get_topology_info()->chip_revision == CPU_VERSION_A0;
#endif
if (kern_feature_override(KF_MATV_OVRD)) {
timebase_validation = 0;
}
if (PE_parse_boot_argn("timebase_validation", &tmp_mv, sizeof(tmp_mv))) {
timebase_validation = tmp_mv;
}
#endif
}
static void
timebase_callback(struct timebase_freq_t * freq)
{
unsigned long numer, denom;
uint64_t t64_1, t64_2;
uint32_t divisor;
if (freq->timebase_den < 1 || freq->timebase_den > 4 ||
freq->timebase_num < freq->timebase_den) {
panic("rtclock timebase_callback: invalid constant %ld / %ld",
freq->timebase_num, freq->timebase_den);
}
denom = freq->timebase_num;
numer = freq->timebase_den * NSEC_PER_SEC;
// reduce by the greatest common denominator to minimize overflow
if (numer > denom) {
t64_1 = numer;
t64_2 = denom;
} else {
t64_1 = denom;
t64_2 = numer;
}
while (t64_2 != 0) {
uint64_t temp = t64_2;
t64_2 = t64_1 % t64_2;
t64_1 = temp;
}
numer /= t64_1;
denom /= t64_1;
rtclock_timebase_const.numer = (uint32_t)numer;
rtclock_timebase_const.denom = (uint32_t)denom;
divisor = (uint32_t)(freq->timebase_num / freq->timebase_den);
rtclock_sec_divisor = divisor;
rtclock_usec_divisor = divisor / USEC_PER_SEC;
}
/*
* Initialize the system clock device for the current cpu
*/
int
rtclock_init(void)
{
uint64_t abstime;
cpu_data_t * cdp;
clock_timebase_init();
cdp = getCpuDatap();
abstime = mach_absolute_time();
cdp->rtcPop = EndOfAllTime; /* Init Pop time */
timer_resync_deadlines(); /* Start the timers going */
return 1;
}
uint64_t
mach_absolute_time(void)
{
#if DEVELOPMENT || DEBUG
if (__improbable(timebase_validation)) {
#if __ARM_ARCH_8_6__ || HAS_ACNTVCT
static _Atomic uint64_t s_last_absolute_time = 1;
uint64_t old_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
/*
* Because this timebase read is nonspeculative, it cannot begin reading
* the timebase value until after the load of the old value completes.
*/
if (old_absolute_time == 0) {
timebase_validation = 0; // we know it's bad, now prevent nested panics
panic("old_absolute_time was 0");
}
uint64_t new_absolute_time = ml_get_timebase();
if (old_absolute_time > new_absolute_time) {
timebase_validation = 0; // prevent nested panics
panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx)",
new_absolute_time, old_absolute_time);
}
if (old_absolute_time < new_absolute_time) {
/* read again, to pretest the atomic max */
uint64_t pretest_absolute_time = os_atomic_load(&s_last_absolute_time, relaxed);
if (pretest_absolute_time < new_absolute_time) {
uint64_t fresh_last_absolute_time = os_atomic_max_orig(&s_last_absolute_time, new_absolute_time, relaxed);
if (fresh_last_absolute_time != pretest_absolute_time) {
/*
* Someone else published a newer time after we loaded s_last_absolute_time.
* Enforce that our timebase is not behind this new one.
* We can't compare it with our previous timebase read, as it is too old.
*/
uint64_t newest_absolute_time = ml_get_timebase();
if (fresh_last_absolute_time > newest_absolute_time) {
timebase_validation = 0; // prevent nested panics
panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old values 0x%llx, 0x%llx, 0x%llx, 0x%llx)\n",
newest_absolute_time, fresh_last_absolute_time, pretest_absolute_time, old_absolute_time, new_absolute_time);
}
}
}
}
return new_absolute_time;
#else /* !(__ARM_ARCH_8_6__ || HAS_ACNTVCT) */
static volatile uint64_t s_last_absolute_time = 0;
uint64_t new_absolute_time, old_absolute_time;
int attempts = 0;
/* ARM 64: We need a dsb here to ensure that the load of s_last_absolute_time
* completes before the timebase read. Were the load to complete after the
* timebase read, there would be a window for another CPU to update
* s_last_absolute_time and leave us in an inconsistent state. Consider the
* following interleaving:
*
* Let s_last_absolute_time = t0
* CPU0: Read timebase at t1
* CPU1: Read timebase at t2
* CPU1: Update s_last_absolute_time to t2
* CPU0: Load completes
* CPU0: Update s_last_absolute_time to t1
*
* This would cause the assertion to fail even though time did not go
* backwards. Thus, we use a dsb to guarantee completion of the load before
* the timebase read.
*/
do {
attempts++;
old_absolute_time = s_last_absolute_time;
__builtin_arm_dsb(DSB_ISHLD);
new_absolute_time = ml_get_timebase();
} while (attempts < MAX_TIMEBASE_TRIES && !OSCompareAndSwap64(old_absolute_time, new_absolute_time, &s_last_absolute_time));
if (attempts < MAX_TIMEBASE_TRIES && old_absolute_time > new_absolute_time) {
timebase_validation = 0; // we know it's bad, now prevent nested panics
panic("mach_absolute_time returning non-monotonically increasing value 0x%llx (old value 0x%llx\n)",
new_absolute_time, old_absolute_time);
}
return new_absolute_time;
#endif /* __ARM_ARCH_8_6__ || HAS_ACNTVCT */
}
#endif /* DEVELOPMENT || DEBUG */
return ml_get_timebase();
}
uint64_t
mach_approximate_time(void)
{
#if __ARM_TIME__ || __ARM_TIME_TIMEBASE_ONLY__ || __arm64__
/* Hardware supports a fast timestamp, so grab it without asserting monotonicity */
return ml_get_timebase();
#else
processor_t processor;
uint64_t approx_time;
disable_preemption();
processor = current_processor();
approx_time = processor->last_dispatch;
enable_preemption();
return approx_time;
#endif
}
void
clock_get_system_microtime(clock_sec_t * secs,
clock_usec_t * microsecs)
{
absolutetime_to_microtime(mach_absolute_time(), secs, microsecs);
}
void
clock_get_system_nanotime(clock_sec_t * secs,
clock_nsec_t * nanosecs)
{
uint64_t abstime;
uint64_t t64;
abstime = mach_absolute_time();
*secs = (t64 = abstime / rtclock_sec_divisor);
abstime -= (t64 * rtclock_sec_divisor);
*nanosecs = (clock_nsec_t)((abstime * NSEC_PER_SEC) / rtclock_sec_divisor);
}
void
clock_gettimeofday_set_commpage(uint64_t abstime,
uint64_t sec,
uint64_t frac,
uint64_t scale,
uint64_t tick_per_sec)
{
commpage_set_timestamp(abstime, sec, frac, scale, tick_per_sec);
}
void
clock_timebase_info(mach_timebase_info_t info)
{
*info = rtclock_timebase_const;
}
/*
* Real-time clock device interrupt.
*/
void
rtclock_intr(__unused unsigned int is_user_context)
{
uint64_t abstime;
cpu_data_t * cdp;
struct arm_saved_state * regs;
unsigned int user_mode;
uintptr_t pc;
cdp = getCpuDatap();
cdp->cpu_stat.timer_cnt++;
SCHED_STATS_INC(timer_pop_count);
assert(!ml_get_interrupts_enabled());
abstime = mach_absolute_time();
if (cdp->cpu_idle_pop != 0x0ULL) {
if ((cdp->rtcPop - abstime) < cdp->cpu_idle_latency) {
cdp->cpu_idle_pop = 0x0ULL;
while (abstime < cdp->rtcPop) {
abstime = mach_absolute_time();
}
} else {
ClearIdlePop(FALSE);
}
}
if ((regs = cdp->cpu_int_state)) {
pc = get_saved_state_pc(regs);
#if __arm64__
user_mode = PSR64_IS_USER(get_saved_state_cpsr(regs));
#endif
} else {
pc = 0;
user_mode = 0;
}
if (abstime >= cdp->rtcPop) {
/* Log the interrupt service latency (-ve value expected by tool) */
KDBG_RELEASE(DECR_TRAP_LATENCY | DBG_FUNC_NONE,
-(abstime - cdp->rtcPop),
user_mode ? pc : VM_KERNEL_UNSLIDE(pc), user_mode);
}
/* call the generic etimer */
timer_intr(user_mode, pc);
}
static int
deadline_to_decrementer(uint64_t deadline,
uint64_t now)
{
uint64_t delt;
if (deadline <= now) {
return DECREMENTER_MIN;
} else {
delt = deadline - now;
return (delt >= (DECREMENTER_MAX + 1)) ? DECREMENTER_MAX : ((delt >= (DECREMENTER_MIN + 1)) ? (int)delt : DECREMENTER_MIN);
}
}
/*
* Request a decrementer pop
*/
int
setPop(uint64_t time)
{
int delay_time;
uint64_t current_time;
cpu_data_t * cdp;
cdp = getCpuDatap();
current_time = mach_absolute_time();
delay_time = deadline_to_decrementer(time, current_time);
cdp->rtcPop = delay_time + current_time;
ml_set_decrementer((uint32_t) delay_time);
return delay_time;
}
/*
* Request decrementer Idle Pop. Return true if set
*/
boolean_t
SetIdlePop(void)
{
int delay_time;
uint64_t time;
uint64_t current_time;
cpu_data_t * cdp;
cdp = getCpuDatap();
current_time = mach_absolute_time();
if (((cdp->rtcPop < current_time) ||
(cdp->rtcPop - current_time) < cdp->cpu_idle_latency)) {
return FALSE;
}
time = cdp->rtcPop - cdp->cpu_idle_latency;
delay_time = deadline_to_decrementer(time, current_time);
cdp->cpu_idle_pop = delay_time + current_time;
ml_set_decrementer((uint32_t) delay_time);
return TRUE;
}
/*
* Clear decrementer Idle Pop
*/
void
ClearIdlePop(
boolean_t wfi)
{
cpu_data_t * cdp;
cdp = getCpuDatap();
cdp->cpu_idle_pop = 0x0ULL;
/*
* Don't update the HW timer if there's a pending
* interrupt (we can lose interrupt assertion);
* we want to take the interrupt right now and update
* the deadline from the handler).
*
* ARM64_TODO: consider this more carefully.
*/
if (!(wfi && ml_get_timer_pending())) {
setPop(cdp->rtcPop);
}
}
void
absolutetime_to_microtime(uint64_t abstime,
clock_sec_t * secs,
clock_usec_t * microsecs)
{
uint64_t t64;
*secs = t64 = abstime / rtclock_sec_divisor;
abstime -= (t64 * rtclock_sec_divisor);
*microsecs = (uint32_t)(abstime / rtclock_usec_divisor);
}
void
absolutetime_to_nanoseconds(uint64_t abstime,
uint64_t * result)
{
uint64_t t64;
*result = (t64 = abstime / rtclock_sec_divisor) * NSEC_PER_SEC;
abstime -= (t64 * rtclock_sec_divisor);
*result += (abstime * NSEC_PER_SEC) / rtclock_sec_divisor;
}
void
nanoseconds_to_absolutetime(uint64_t nanosecs,
uint64_t * result)
{
uint64_t t64;
*result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
nanosecs -= (t64 * NSEC_PER_SEC);
*result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}
void
nanotime_to_absolutetime(clock_sec_t secs,
clock_nsec_t nanosecs,
uint64_t * result)
{
*result = ((uint64_t) secs * rtclock_sec_divisor) +
((uint64_t) nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}
void
clock_interval_to_absolutetime_interval(uint32_t interval,
uint32_t scale_factor,
uint64_t * result)
{
uint64_t nanosecs = (uint64_t) interval * scale_factor;
uint64_t t64;
*result = (t64 = nanosecs / NSEC_PER_SEC) * rtclock_sec_divisor;
nanosecs -= (t64 * NSEC_PER_SEC);
*result += (nanosecs * rtclock_sec_divisor) / NSEC_PER_SEC;
}
void
machine_delay_until(uint64_t interval,
uint64_t deadline)
{
#pragma unused(interval)
uint64_t now;
do {
__builtin_arm_wfe();
now = mach_absolute_time();
} while (now < deadline);
}