/*
 * Copyright (c) 2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/**
 * This file is meant to contain all of the PPL entry points and PPL-specific
 * functionality.
 *
 * Every single function in the pmap that chooses between running a "*_ppl()" or
 * "*_internal()" function variant will be placed into this file. This file also
 * contains the ppl_handler_table, as well as a few PPL-only entry/exit helper
 * functions.
 *
 * See doc/arm/PPL.md for more information about how these PPL entry points work.
 */
#include <kern/ledger.h>

#include <vm/vm_object_xnu.h>
#include <vm/vm_page_internal.h>
#include <vm/vm_pageout_xnu.h>

#include <arm/pmap/pmap_internal.h>

/**
 * Keeps track of the total number of pages taken from the PPL page free lists
 * and returned back to the kernel. This value isn't used in logic anywhere,
 * it's available for debugging purposes strictly.
 */
#if XNU_MONITOR
static uint64_t pmap_ppl_pages_returned_to_kernel_count_total = 0;
#endif /* XNU_MONITOR */

/**
 * PMAP_SUPPORT_PROTOTYPES() will automatically create prototypes for the
 * _internal() and _ppl() variants of a PPL entry point. It also automatically
 * generates the code for the _ppl() variant which is what is used to jump into
 * the PPL.
 *
 * See doc/arm/PPL.md for more information about how these PPL entry points work.
 */

#if XNU_MONITOR

PMAP_SUPPORT_PROTOTYPES(
	void,
	pmap_mark_page_as_ppl_page, (pmap_paddr_t pa, bool initially_free), PMAP_MARK_PAGE_AS_PMAP_PAGE_INDEX);

PMAP_SUPPORT_PROTOTYPES(
	void,
	pmap_cpu_data_init, (unsigned int cpu_number), PMAP_CPU_DATA_INIT_INDEX);

PMAP_SUPPORT_PROTOTYPES(
	uint64_t,
	pmap_release_ppl_pages_to_kernel, (void), PMAP_RELEASE_PAGES_TO_KERNEL_INDEX);

PMAP_SUPPORT_PROTOTYPES(
	void,
	pmap_ledger_verify_size, (size_t),
	PMAP_LEDGER_VERIFY_SIZE_INDEX);

PMAP_SUPPORT_PROTOTYPES(
	ledger_t,
	pmap_ledger_alloc, (void),
	PMAP_LEDGER_ALLOC_INDEX);

PMAP_SUPPORT_PROTOTYPES(
	void,
	pmap_ledger_free, (ledger_t),
	PMAP_LEDGER_FREE_INDEX);

#endif /* XNU_MONITOR */

PMAP_SUPPORT_PROTOTYPES(
	kern_return_t,
	mapping_free_prime, (void), MAPPING_FREE_PRIME_INDEX);

/* TODO: Move the ppl_handler_table into this file. */

#if XNU_MONITOR

/**
 * Claim a page on behalf of the PPL by marking it as PPL-owned and only
 * allowing the PPL to write to it. Also adds that page to the PPL page free
 * list for allocation later.
 *
 * @param pa The physical address of the page to mark as PPL-owned.
 */
void
pmap_mark_page_as_ppl_page(pmap_paddr_t pa)
{
	pmap_mark_page_as_ppl_page_ppl(pa, true);
}

/**
 * Quickly release pages living on the PPL page free list back to the VM. The
 * VM will call this when the system is under memory pressure.
 *
 * @note A minimum amount of pages (set by PMAP_MIN_FREE_PPL_PAGES) will always
 *       be kept on the PPL page free list to ensure that core operations can
 *       occur without having to refill the free list.
 */
uint64_t
pmap_release_ppl_pages_to_kernel(void)
{
	pmap_paddr_t pa = 0;
	vm_page_t mem = VM_PAGE_NULL;
	vm_page_t local_freeq = VM_PAGE_NULL;
	uint64_t pmap_ppl_pages_returned_to_kernel_count = 0;

	while (pmap_ppl_free_page_count > PMAP_MIN_FREE_PPL_PAGES) {
		/* Convert a single PPL page back into a kernel-usable page. */
		pa = pmap_release_ppl_pages_to_kernel_ppl();

		if (!pa) {
			break;
		}

		/**
		 * If we retrieved a page, add it to the queue of pages that will be
		 * given back to the VM.
		 */
		vm_object_lock(pmap_object);

		mem = vm_page_lookup(pmap_object, (pa - gPhysBase));
		assert(mem != VM_PAGE_NULL);
		assert(VM_PAGE_WIRED(mem));

		mem->vmp_busy = TRUE;
		mem->vmp_snext = local_freeq;
		local_freeq = mem;
		pmap_ppl_pages_returned_to_kernel_count++;
		pmap_ppl_pages_returned_to_kernel_count_total++;

		/* Pages are considered "in use" until given back to the VM. */
		OSAddAtomic(-1, &inuse_pmap_pages_count);

		vm_object_unlock(pmap_object);
	}

	/**
	 * Return back the pages to the VM that we've converted into kernel-usable
	 * pages.
	 */
	if (local_freeq) {
		/* We need to hold the object lock for freeing pages. */
		vm_object_lock(pmap_object);
		vm_page_free_list(local_freeq, TRUE);
		vm_object_unlock(pmap_object);
	}

	/**
	 * If we have any pages to return to the VM, take the page queues lock and
	 * decrement the wire count.
	 */
	if (pmap_ppl_pages_returned_to_kernel_count) {
		vm_page_lockspin_queues();
		vm_page_wire_count -= pmap_ppl_pages_returned_to_kernel_count;
		vm_page_unlock_queues();
	}

	return pmap_ppl_pages_returned_to_kernel_count;
}

#endif /* XNU_MONITOR */

/**
 * See pmap_cpu_data_init_internal()'s function header for more info.
 */
void
pmap_cpu_data_init(void)
{
#if XNU_MONITOR
	pmap_cpu_data_init_ppl(cpu_number());
#else
	pmap_cpu_data_init_internal(cpu_number());
#endif
}

/**
 * Prime the pv_entry_t free lists with a healthy amount of objects first thing
 * during boot. These objects will be used to keep track of physical-to-virtual
 * mappings.
 */
void
mapping_free_prime(void)
{
	kern_return_t kr = KERN_FAILURE;

#if XNU_MONITOR
	unsigned int i = 0;

	/**
	 * Allocate the needed PPL pages up front, to minimize the chance that we
	 * will need to call into the PPL multiple times.
	 */
	for (i = 0; i < pv_alloc_initial_target; i += (PAGE_SIZE / sizeof(pv_entry_t))) {
		pmap_alloc_page_for_ppl(0);
	}

	for (i = 0; i < pv_kern_alloc_initial_target; i += (PAGE_SIZE / sizeof(pv_entry_t))) {
		pmap_alloc_page_for_ppl(0);
	}

	while ((kr = mapping_free_prime_ppl()) == KERN_RESOURCE_SHORTAGE) {
		pmap_alloc_page_for_ppl(0);
	}
#else /* XNU_MONITOR */
	kr = mapping_free_prime_internal();
#endif /* XNU_MONITOR */

	if (kr != KERN_SUCCESS) {
		panic("%s: failed, no pages available? kr=%d", __func__, kr);
	}
}

/**
 * See pmap_ledger_verify_size_internal()'s function header for more information.
 */
#if !XNU_MONITOR
__attribute__((noreturn))
#endif /* !XNU_MONITOR */
void
pmap_ledger_verify_size(size_t size)
{
#if XNU_MONITOR
	pmap_ledger_verify_size_ppl(size);
#else /* XNU_MONITOR */
	/**
	 * Ledger objects are only managed by the pmap on PPL-enabled systems. Other
	 * systems will allocate them using a zone allocator.
	 */
	panic("%s: unsupported on non-PPL systems, size=%lu", __func__, size);
	__builtin_unreachable();
#endif /* XNU_MONITOR */
}

/**
 * See pmap_ledger_alloc_internal()'s function header for more information.
 */
ledger_t
pmap_ledger_alloc(void)
{
#if XNU_MONITOR
	ledger_t ledger = NULL;

	while ((ledger = pmap_ledger_alloc_ppl()) == NULL) {
		pmap_alloc_page_for_ppl(0);
	}

	return ledger;
#else /* XNU_MONITOR */
	/**
	 * Ledger objects are only managed by the pmap on PPL-enabled systems. Other
	 * systems will allocate them using a zone allocator.
	 */
	panic("%s: unsupported on non-PPL systems", __func__);
	__builtin_unreachable();
#endif /* XNU_MONITOR */
}

/**
 * See pmap_ledger_free_internal()'s function header for more information.
 */
#if !XNU_MONITOR
__attribute__((noreturn))
#endif /* !XNU_MONITOR */
void
pmap_ledger_free(ledger_t ledger)
{
#if XNU_MONITOR
	pmap_ledger_free_ppl(ledger);
#else /* XNU_MONITOR */
	/**
	 * Ledger objects are only managed by the pmap on PPL-enabled systems. Other
	 * systems will allocate them using a zone allocator.
	 */
	panic("%s: unsupported on non-PPL systems, ledger=%p", __func__, ledger);
	__builtin_unreachable();
#endif /* XNU_MONITOR */
}
