This is xnu-11215.1.10. See this file in:
/*
* Copyright (c) 2000-2021 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
/*
* Mach Operating System
* Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
* All Rights Reserved.
*
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
*
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
*
* Carnegie Mellon requests users of this software to return to
*
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
*
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
/*
*/
/*
* File: vm/vm_user.c
* Author: Avadis Tevanian, Jr., Michael Wayne Young
*
* User-exported virtual memory functions.
*/
/*
* There are three implementations of the "XXX_allocate" functionality in
* the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
* (for a task with the same address space size, especially the current task),
* and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
* in the kernel should only be used on the kernel_task. vm32_vm_allocate only
* makes sense on platforms where a user task can either be 32 or 64, or the kernel
* task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
* for new code.
*
* The entrypoints into the kernel are more complex. All platforms support a
* mach_vm_allocate-style API (subsystem 4800) which operates with the largest
* size types for the platform. On platforms that only support U32/K32,
* subsystem 4800 is all you need. On platforms that support both U32 and U64,
* subsystem 3800 is used disambiguate the size of parameters, and they will
* always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
* the MIG glue should never call into vm_allocate directly, because the calling
* task and kernel_task are unlikely to use the same size parameters
*
* New VM call implementations should be added here and to mach_vm.defs
* (subsystem 4800), and use mach_vm_* "wide" types.
*/
#include <debug.h>
#include <mach/boolean.h>
#include <mach/kern_return.h>
#include <mach/mach_types.h> /* to get vm_address_t */
#include <mach/memory_object.h>
#include <mach/std_types.h> /* to get pointer_t */
#include <mach/upl.h>
#include <mach/vm_attributes.h>
#include <mach/vm_param.h>
#include <mach/vm_statistics.h>
#include <mach/mach_syscalls.h>
#include <mach/sdt.h>
#include <mach/memory_entry.h>
#include <mach/host_priv_server.h>
#include <mach/mach_vm_server.h>
#include <mach/memory_entry_server.h>
#include <mach/vm_map_server.h>
#include <kern/host.h>
#include <kern/kalloc.h>
#include <kern/task.h>
#include <kern/misc_protos.h>
#include <vm/vm_fault.h>
#include <vm/vm_map_internal.h>
#include <vm/vm_object_xnu.h>
#include <vm/vm_kern.h>
#include <vm/vm_page_internal.h>
#include <vm/memory_object_internal.h>
#include <vm/vm_pageout_internal.h>
#include <vm/vm_protos.h>
#include <vm/vm_purgeable_internal.h>
#include <vm/vm_memory_entry_xnu.h>
#include <vm/vm_kern_internal.h>
#include <vm/vm_iokit.h>
#include <vm/vm_sanitize_internal.h>
#if CONFIG_DEFERRED_RECLAIM
#include <vm/vm_reclaim_internal.h>
#endif /* CONFIG_DEFERRED_RECLAIM */
#include <vm/vm_init_xnu.h>
#include <san/kasan.h>
#include <libkern/OSDebug.h>
#include <IOKit/IOBSD.h>
#include <sys/kdebug_triage.h>
/*
* mach_vm_allocate allocates "zero fill" memory in the specfied
* map.
*/
kern_return_t
mach_vm_allocate_external(
vm_map_t map,
mach_vm_offset_ut *addr,
mach_vm_size_ut size,
int flags)
{
vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
/* filter out any kernel-only flags */
if (flags & ~VM_FLAGS_USER_ALLOCATE) {
ktriage_record(thread_tid(current_thread()),
KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM,
KDBG_TRIAGE_RESERVED,
KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADFLAGS_ERROR),
KERN_INVALID_ARGUMENT /* arg */);
return KERN_INVALID_ARGUMENT;
}
vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
return mach_vm_allocate_kernel(map, addr, size, vmk_flags);
}
/*
* vm_allocate
* Legacy routine that allocates "zero fill" memory in the specfied
* map (which is limited to the same size as the kernel).
*/
kern_return_t
vm_allocate_external(
vm_map_t map,
vm_offset_ut *addr,
vm_size_ut size,
int flags)
{
return mach_vm_allocate_external(map, addr, size, flags);
}
static inline kern_return_t
mach_vm_deallocate_sanitize(
vm_map_t map,
mach_vm_offset_ut start_u,
mach_vm_size_ut size_u,
mach_vm_offset_t *start,
mach_vm_offset_t *end,
mach_vm_size_t *size)
{
return vm_sanitize_addr_size(start_u, size_u, VM_SANITIZE_CALLER_VM_DEALLOCATE,
map, VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS, start,
end, size);
}
/*
* mach_vm_deallocate -
* deallocates the specified range of addresses in the
* specified address map.
*/
kern_return_t
mach_vm_deallocate(
vm_map_t map,
mach_vm_offset_ut start_u,
mach_vm_size_ut size_u)
{
mach_vm_offset_t start, end;
mach_vm_size_t size;
kern_return_t kr;
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
kr = mach_vm_deallocate_sanitize(map,
start_u,
size_u,
&start,
&end,
&size);
if (__improbable(kr != KERN_SUCCESS)) {
return vm_sanitize_get_kr(kr);
}
return vm_map_remove_guard(map, start, end,
VM_MAP_REMOVE_NO_FLAGS,
KMEM_GUARD_NONE).kmr_return;
}
/*
* vm_deallocate -
* deallocates the specified range of addresses in the
* specified address map (limited to addresses the same
* size as the kernel).
*/
kern_return_t
vm_deallocate(
vm_map_t map,
vm_offset_ut start,
vm_size_ut size)
{
return mach_vm_deallocate(map, start, size);
}
/*
* mach_vm_inherit -
* Sets the inheritance of the specified range in the
* specified map.
*/
kern_return_t
mach_vm_inherit(
vm_map_t map,
mach_vm_offset_t start,
mach_vm_size_t size,
vm_inherit_t new_inheritance)
{
if ((map == VM_MAP_NULL) || (start + size < start) ||
(new_inheritance > VM_INHERIT_LAST_VALID)) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_inherit(map,
vm_map_trunc_page(start,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(start + size,
VM_MAP_PAGE_MASK(map)),
new_inheritance);
}
/*
* vm_inherit -
* Sets the inheritance of the specified range in the
* specified map (range limited to addresses
*/
kern_return_t
vm_inherit(
vm_map_t map,
vm_offset_t start,
vm_size_t size,
vm_inherit_t new_inheritance)
{
if ((map == VM_MAP_NULL) || (start + size < start) ||
(new_inheritance > VM_INHERIT_LAST_VALID)) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_inherit(map,
vm_map_trunc_page(start,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(start + size,
VM_MAP_PAGE_MASK(map)),
new_inheritance);
}
/*
* mach_vm_protect -
* Sets the protection of the specified range in the
* specified map.
*/
kern_return_t
mach_vm_protect(
vm_map_t map,
mach_vm_offset_t start,
mach_vm_size_t size,
boolean_t set_maximum,
vm_prot_t new_protection)
{
if ((map == VM_MAP_NULL) || (start + size < start) ||
(new_protection & ~(VM_PROT_ALL | VM_PROT_COPY))) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_protect(map,
vm_map_trunc_page(start,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(start + size,
VM_MAP_PAGE_MASK(map)),
new_protection,
set_maximum);
}
/*
* vm_protect -
* Sets the protection of the specified range in the
* specified map. Addressability of the range limited
* to the same size as the kernel.
*/
kern_return_t
vm_protect(
vm_map_t map,
vm_offset_t start,
vm_size_t size,
boolean_t set_maximum,
vm_prot_t new_protection)
{
if ((map == VM_MAP_NULL) || (start + size < start) ||
(new_protection & ~VM_VALID_VMPROTECT_FLAGS)
#if defined(__x86_64__)
|| ((new_protection & VM_PROT_UEXEC) && !pmap_supported_feature(map->pmap, PMAP_FEAT_UEXEC))
#endif
) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_protect(map,
vm_map_trunc_page(start,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(start + size,
VM_MAP_PAGE_MASK(map)),
new_protection,
set_maximum);
}
/*
* mach_vm_machine_attributes -
* Handle machine-specific attributes for a mapping, such
* as cachability, migrability, etc.
*/
kern_return_t
mach_vm_machine_attribute(
vm_map_t map,
mach_vm_address_t addr,
mach_vm_size_t size,
vm_machine_attribute_t attribute,
vm_machine_attribute_val_t* value) /* IN/OUT */
{
if ((map == VM_MAP_NULL) || (addr + size < addr)) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_machine_attribute(
map,
vm_map_trunc_page(addr,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(addr + size,
VM_MAP_PAGE_MASK(map)),
attribute,
value);
}
/*
* vm_machine_attribute -
* Handle machine-specific attributes for a mapping, such
* as cachability, migrability, etc. Limited addressability
* (same range limits as for the native kernel map).
*/
kern_return_t
vm_machine_attribute(
vm_map_t map,
vm_address_t addr,
vm_size_t size,
vm_machine_attribute_t attribute,
vm_machine_attribute_val_t* value) /* IN/OUT */
{
if ((map == VM_MAP_NULL) || (addr + size < addr)) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
return vm_map_machine_attribute(
map,
vm_map_trunc_page(addr,
VM_MAP_PAGE_MASK(map)),
vm_map_round_page(addr + size,
VM_MAP_PAGE_MASK(map)),
attribute,
value);
}
/*
* mach_vm_read -
* Read/copy a range from one address space and return it to the caller.
*
* It is assumed that the address for the returned memory is selected by
* the IPC implementation as part of receiving the reply to this call.
* If IPC isn't used, the caller must deal with the vm_map_copy_t object
* that gets returned.
*
* JMM - because of mach_msg_type_number_t, this call is limited to a
* single 4GB region at this time.
*
*/
kern_return_t
mach_vm_read(
vm_map_t map,
mach_vm_address_ut addr,
mach_vm_size_ut size,
pointer_t *data,
mach_msg_type_number_t *data_size)
{
kern_return_t error;
vm_map_copy_t ipc_address;
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
/*
* mach_msg_type_number_t is a signed int,
* make sure we do not overflow it.
*/
if (!VM_SANITIZE_UNSAFE_FITS(size, mach_msg_type_number_t)) {
return KERN_INVALID_ARGUMENT;
}
error = vm_map_copyin(map, addr, size, FALSE, &ipc_address);
if (KERN_SUCCESS == error) {
*data = (pointer_t) ipc_address;
*data_size = (mach_msg_type_number_t)VM_SANITIZE_UNSAFE_UNWRAP(size);
}
return error;
}
/*
* vm_read -
* Read/copy a range from one address space and return it to the caller.
* Limited addressability (same range limits as for the native kernel map).
*
* It is assumed that the address for the returned memory is selected by
* the IPC implementation as part of receiving the reply to this call.
* If IPC isn't used, the caller must deal with the vm_map_copy_t object
* that gets returned.
*/
kern_return_t
vm_read(
vm_map_t map,
vm_address_ut addr,
vm_size_ut size,
pointer_t *data,
mach_msg_type_number_t *data_size)
{
return mach_vm_read(map, addr, size, data, data_size);
}
/*
* mach_vm_read_list -
* Read/copy a list of address ranges from specified map.
*
* MIG does not know how to deal with a returned array of
* vm_map_copy_t structures, so we have to do the copyout
* manually here.
*/
kern_return_t
mach_vm_read_list(
vm_map_t map,
mach_vm_read_entry_t data_list,
natural_t count)
{
mach_msg_type_number_t i;
kern_return_t error;
vm_map_copy_t copy;
if (map == VM_MAP_NULL ||
count > VM_MAP_ENTRY_MAX) {
return KERN_INVALID_ARGUMENT;
}
error = KERN_SUCCESS;
for (i = 0; i < count; i++) {
vm_map_address_t map_addr;
vm_map_size_t map_size;
map_addr = (vm_map_address_t)(data_list[i].address);
map_size = (vm_map_size_t)(data_list[i].size);
if (map_size != 0) {
error = vm_map_copyin(map,
map_addr,
map_size,
FALSE, /* src_destroy */
©);
if (KERN_SUCCESS == error) {
error = vm_map_copyout(
current_task()->map,
&map_addr,
copy);
if (KERN_SUCCESS == error) {
data_list[i].address = map_addr;
continue;
}
vm_map_copy_discard(copy);
}
}
data_list[i].address = (mach_vm_address_t)0;
data_list[i].size = (mach_vm_size_t)0;
}
return error;
}
/*
* vm_read_list -
* Read/copy a list of address ranges from specified map.
*
* MIG does not know how to deal with a returned array of
* vm_map_copy_t structures, so we have to do the copyout
* manually here.
*
* The source and destination ranges are limited to those
* that can be described with a vm_address_t (i.e. same
* size map as the kernel).
*
* JMM - If the result of the copyout is an address range
* that cannot be described with a vm_address_t (i.e. the
* caller had a larger address space but used this call
* anyway), it will result in a truncated address being
* returned (and a likely confused caller).
*/
kern_return_t
vm_read_list(
vm_map_t map,
vm_read_entry_t data_list,
natural_t count)
{
mach_msg_type_number_t i;
kern_return_t error;
vm_map_copy_t copy;
if (map == VM_MAP_NULL ||
count > VM_MAP_ENTRY_MAX) {
return KERN_INVALID_ARGUMENT;
}
error = KERN_SUCCESS;
for (i = 0; i < count; i++) {
vm_map_address_t map_addr;
vm_map_size_t map_size;
map_addr = (vm_map_address_t)(data_list[i].address);
map_size = (vm_map_size_t)(data_list[i].size);
if (map_size != 0) {
error = vm_map_copyin(map,
map_addr,
map_size,
FALSE, /* src_destroy */
©);
if (KERN_SUCCESS == error) {
error = vm_map_copyout(current_task()->map,
&map_addr,
copy);
if (KERN_SUCCESS == error) {
data_list[i].address =
CAST_DOWN(vm_offset_t, map_addr);
continue;
}
vm_map_copy_discard(copy);
}
}
data_list[i].address = (mach_vm_address_t)0;
data_list[i].size = (mach_vm_size_t)0;
}
return error;
}
/*
* mach_vm_read_overwrite -
* Overwrite a range of the current map with data from the specified
* map/address range.
*
* In making an assumption that the current thread is local, it is
* no longer cluster-safe without a fully supportive local proxy
* thread/task (but we don't support cluster's anymore so this is moot).
*/
kern_return_t
mach_vm_read_overwrite(
vm_map_t map,
mach_vm_address_ut address,
mach_vm_size_ut size,
mach_vm_address_ut data,
mach_vm_size_ut *data_size)
{
kern_return_t error;
vm_map_copy_t copy;
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
error = vm_map_copyin(map, address, size, FALSE, ©);
if (KERN_SUCCESS == error) {
if (copy) {
assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
}
error = vm_map_copy_overwrite(current_thread()->map,
data, copy, size, FALSE);
if (KERN_SUCCESS == error) {
*data_size = size;
return error;
}
vm_map_copy_discard(copy);
}
return error;
}
/*
* vm_read_overwrite -
* Overwrite a range of the current map with data from the specified
* map/address range.
*
* This routine adds the additional limitation that the source and
* destination ranges must be describable with vm_address_t values
* (i.e. the same size address spaces as the kernel, or at least the
* the ranges are in that first portion of the respective address
* spaces).
*/
kern_return_t
vm_read_overwrite(
vm_map_t map,
vm_address_ut address,
vm_size_ut size,
vm_address_ut data,
vm_size_ut *data_size)
{
return mach_vm_read_overwrite(map, address, size, data, data_size);
}
/*
* mach_vm_write -
* Overwrite the specified address range with the data provided
* (from the current map).
*/
kern_return_t
mach_vm_write(
vm_map_t map,
mach_vm_address_ut address,
pointer_t data,
mach_msg_type_number_t size)
{
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
return vm_map_copy_overwrite(map, address,
(vm_map_copy_t) data, size, FALSE /* interruptible XXX */);
}
/*
* vm_write -
* Overwrite the specified address range with the data provided
* (from the current map).
*
* The addressability of the range of addresses to overwrite is
* limited bu the use of a vm_address_t (same size as kernel map).
* Either the target map is also small, or the range is in the
* low addresses within it.
*/
kern_return_t
vm_write(
vm_map_t map,
vm_address_ut address,
pointer_t data,
mach_msg_type_number_t size)
{
return mach_vm_write(map, address, data, size);
}
/*
* mach_vm_copy -
* Overwrite one range of the specified map with the contents of
* another range within that same map (i.e. both address ranges
* are "over there").
*/
kern_return_t
mach_vm_copy(
vm_map_t map,
mach_vm_address_ut source_address,
mach_vm_size_ut size,
mach_vm_address_ut dest_address)
{
vm_map_copy_t copy;
kern_return_t kr;
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
kr = vm_map_copyin(map, source_address, size, FALSE, ©);
if (KERN_SUCCESS == kr) {
if (copy) {
assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
}
kr = vm_map_copy_overwrite(map, dest_address,
copy, size, FALSE /* interruptible XXX */);
if (KERN_SUCCESS != kr) {
vm_map_copy_discard(copy);
}
}
return kr;
}
kern_return_t
vm_copy(
vm_map_t map,
vm_address_ut source_address,
vm_size_ut size,
vm_address_ut dest_address)
{
return mach_vm_copy(map, source_address, size, dest_address);
}
/*
* mach_vm_map -
* Map some range of an object into an address space.
*
* The object can be one of several types of objects:
* NULL - anonymous memory
* a named entry - a range within another address space
* or a range within a memory object
* a whole memory object
*
*/
kern_return_t
mach_vm_map_external(
vm_map_t target_map,
mach_vm_offset_ut *address,
mach_vm_size_ut initial_size,
mach_vm_offset_ut mask,
int flags,
ipc_port_t port,
memory_object_offset_ut offset,
boolean_t copy,
vm_prot_ut cur_protection,
vm_prot_ut max_protection,
vm_inherit_ut inheritance)
{
vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
/* filter out any kernel-only flags */
if (flags & ~VM_FLAGS_USER_MAP) {
return KERN_INVALID_ARGUMENT;
}
vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
/* range_id is set by mach_vm_map_kernel */
return mach_vm_map_kernel(target_map, address, initial_size, mask,
vmk_flags, port, offset, copy,
cur_protection, max_protection,
inheritance);
}
/* legacy interface */
__attribute__((always_inline))
kern_return_t
vm_map_64_external(
vm_map_t target_map,
vm_offset_ut *address,
vm_size_ut size,
vm_offset_ut mask,
int flags,
ipc_port_t port,
memory_object_offset_ut offset,
boolean_t copy,
vm_prot_ut cur_protection,
vm_prot_ut max_protection,
vm_inherit_ut inheritance)
{
return mach_vm_map_external(target_map, address,
size, mask, flags, port, offset, copy,
cur_protection, max_protection, inheritance);
}
/* temporary, until world build */
__attribute__((always_inline))
kern_return_t
vm_map_external(
vm_map_t target_map,
vm_offset_ut *address,
vm_size_ut size,
vm_offset_ut mask,
int flags,
ipc_port_t port,
vm_offset_ut offset,
boolean_t copy,
vm_prot_ut cur_protection,
vm_prot_ut max_protection,
vm_inherit_ut inheritance)
{
return mach_vm_map_external(target_map, address,
size, mask, flags, port, offset, copy,
cur_protection, max_protection, inheritance);
}
static inline kern_return_t
mach_vm_remap_new_external_sanitize(
vm_map_t target_map,
vm_prot_ut cur_protection_u,
vm_prot_ut max_protection_u,
vm_prot_t *cur_protection,
vm_prot_t *max_protection)
{
return vm_sanitize_cur_and_max_prots(cur_protection_u, max_protection_u,
VM_SANITIZE_CALLER_VM_MAP_REMAP, target_map,
cur_protection, max_protection);
}
/*
* mach_vm_remap_new -
* Behaves like mach_vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
* and {cur,max}_protection are in/out.
*/
kern_return_t
mach_vm_remap_new_external(
vm_map_t target_map,
mach_vm_offset_ut *address,
mach_vm_size_ut size,
mach_vm_offset_ut mask,
int flags,
mach_port_t src_tport,
mach_vm_offset_ut memory_address,
boolean_t copy,
vm_prot_ut *cur_protection_u, /* IN/OUT */
vm_prot_ut *max_protection_u, /* IN/OUT */
vm_inherit_ut inheritance)
{
vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
vm_map_t src_map;
vm_prot_t cur_protection, max_protection;
kern_return_t kr;
if (target_map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
/* filter out any kernel-only flags */
if (flags & ~VM_FLAGS_USER_REMAP) {
return KERN_INVALID_ARGUMENT;
}
vm_map_kernel_flags_set_vmflags(&vmk_flags,
flags | VM_FLAGS_RETURN_DATA_ADDR);
/*
* We don't need cur_protection here, but sanitizing it before
* enforcing W^X below matches historical error codes better.
*/
kr = mach_vm_remap_new_external_sanitize(target_map,
*cur_protection_u,
*max_protection_u,
&cur_protection,
&max_protection);
if (__improbable(kr != KERN_SUCCESS)) {
return vm_sanitize_get_kr(kr);
}
if ((max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
(VM_PROT_WRITE | VM_PROT_EXECUTE)) {
/*
* XXX FBDP TODO
* enforce target's "wx" policies
*/
return KERN_PROTECTION_FAILURE;
}
if (copy || max_protection == VM_PROT_READ || max_protection == VM_PROT_NONE) {
src_map = convert_port_to_map_read(src_tport);
} else {
src_map = convert_port_to_map(src_tport);
}
/* range_id is set by vm_map_remap */
kr = vm_map_remap(target_map,
address,
size,
mask,
vmk_flags,
src_map,
memory_address,
copy,
cur_protection_u, /* IN/OUT */
max_protection_u, /* IN/OUT */
inheritance);
vm_map_deallocate(src_map);
if (kr == KERN_SUCCESS) {
ipc_port_release_send(src_tport); /* consume on success */
}
return kr;
}
/*
* mach_vm_remap -
* Remap a range of memory from one task into another,
* to another address range within the same task, or
* over top of itself (with altered permissions and/or
* as an in-place copy of itself).
*/
kern_return_t
mach_vm_remap_external(
vm_map_t target_map,
mach_vm_offset_ut *address,
mach_vm_size_ut size,
mach_vm_offset_ut mask,
int flags,
vm_map_t src_map,
mach_vm_offset_ut memory_address,
boolean_t copy,
vm_prot_ut *cur_protection, /* OUT */
vm_prot_ut *max_protection, /* OUT */
vm_inherit_ut inheritance)
{
vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
/* filter out any kernel-only flags */
if (flags & ~VM_FLAGS_USER_REMAP) {
return KERN_INVALID_ARGUMENT;
}
vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
*cur_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
*max_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
/* range_id is set by vm_map_remap */
return vm_map_remap(target_map,
address,
size,
mask,
vmk_flags,
src_map,
memory_address,
copy,
cur_protection,
max_protection,
inheritance);
}
/*
* vm_remap_new -
* Behaves like vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
* and {cur,max}_protection are in/out.
*/
kern_return_t
vm_remap_new_external(
vm_map_t target_map,
vm_offset_ut *address,
vm_size_ut size,
vm_offset_ut mask,
int flags,
mach_port_t src_tport,
vm_offset_ut memory_address,
boolean_t copy,
vm_prot_ut *cur_protection, /* IN/OUT */
vm_prot_ut *max_protection, /* IN/OUT */
vm_inherit_ut inheritance)
{
return mach_vm_remap_new_external(target_map,
address,
size,
mask,
flags,
src_tport,
memory_address,
copy,
cur_protection, /* IN/OUT */
max_protection, /* IN/OUT */
inheritance);
}
/*
* vm_remap -
* Remap a range of memory from one task into another,
* to another address range within the same task, or
* over top of itself (with altered permissions and/or
* as an in-place copy of itself).
*
* The addressability of the source and target address
* range is limited by the size of vm_address_t (in the
* kernel context).
*/
kern_return_t
vm_remap_external(
vm_map_t target_map,
vm_offset_ut *address,
vm_size_ut size,
vm_offset_ut mask,
int flags,
vm_map_t src_map,
vm_offset_ut memory_address,
boolean_t copy,
vm_prot_ut *cur_protection, /* OUT */
vm_prot_ut *max_protection, /* OUT */
vm_inherit_ut inheritance)
{
return mach_vm_remap_external(target_map, address,
size, mask, flags, src_map, memory_address, copy,
cur_protection, max_protection, inheritance);
}
/*
* NOTE: these routine (and this file) will no longer require mach_host_server.h
* when mach_vm_wire and vm_wire are changed to use ledgers.
*/
#include <mach/mach_host_server.h>
/*
* mach_vm_wire
* Specify that the range of the virtual address space
* of the target task must not cause page faults for
* the indicated accesses.
*
* [ To unwire the pages, specify VM_PROT_NONE. ]
*/
kern_return_t
mach_vm_wire_external(
host_priv_t host_priv,
vm_map_t map,
mach_vm_address_ut start,
mach_vm_size_ut size,
vm_prot_ut access)
{
kern_return_t rc;
mach_vm_offset_ut end;
if (host_priv == HOST_PRIV_NULL) {
return KERN_INVALID_HOST;
}
if (map == VM_MAP_NULL) {
return KERN_INVALID_TASK;
}
end = vm_sanitize_compute_unsafe_end(start, size);
if (VM_SANITIZE_UNSAFE_IS_ZERO(access)) {
rc = vm_map_unwire_impl(map, start, end, true,
VM_SANITIZE_CALLER_VM_UNWIRE_USER);
} else {
rc = vm_map_wire_impl(map, start, end, access,
VM_KERN_MEMORY_MLOCK, true, NULL, VM_SANITIZE_CALLER_VM_WIRE_USER);
}
return rc;
}
/*
* vm_wire -
* Specify that the range of the virtual address space
* of the target task must not cause page faults for
* the indicated accesses.
*
* [ To unwire the pages, specify VM_PROT_NONE. ]
*/
kern_return_t
vm_wire(
host_priv_t host_priv,
vm_map_t map,
vm_offset_ut start,
vm_size_ut size,
vm_prot_ut access)
{
return mach_vm_wire_external(host_priv, map, start, size, access);
}
/*
* vm_msync
*
* Synchronises the memory range specified with its backing store
* image by either flushing or cleaning the contents to the appropriate
* memory manager.
*
* interpretation of sync_flags
* VM_SYNC_INVALIDATE - discard pages, only return precious
* pages to manager.
*
* VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
* - discard pages, write dirty or precious
* pages back to memory manager.
*
* VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
* - write dirty or precious pages back to
* the memory manager.
*
* VM_SYNC_CONTIGUOUS - does everything normally, but if there
* is a hole in the region, and we would
* have returned KERN_SUCCESS, return
* KERN_INVALID_ADDRESS instead.
*
* RETURNS
* KERN_INVALID_TASK Bad task parameter
* KERN_INVALID_ARGUMENT both sync and async were specified.
* KERN_SUCCESS The usual.
* KERN_INVALID_ADDRESS There was a hole in the region.
*/
kern_return_t
mach_vm_msync(
vm_map_t map,
mach_vm_address_t address,
mach_vm_size_t size,
vm_sync_t sync_flags)
{
if (map == VM_MAP_NULL) {
return KERN_INVALID_TASK;
}
return vm_map_msync(map, (vm_map_address_t)address,
(vm_map_size_t)size, sync_flags);
}
/*
* vm_msync
*
* Synchronises the memory range specified with its backing store
* image by either flushing or cleaning the contents to the appropriate
* memory manager.
*
* interpretation of sync_flags
* VM_SYNC_INVALIDATE - discard pages, only return precious
* pages to manager.
*
* VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
* - discard pages, write dirty or precious
* pages back to memory manager.
*
* VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
* - write dirty or precious pages back to
* the memory manager.
*
* VM_SYNC_CONTIGUOUS - does everything normally, but if there
* is a hole in the region, and we would
* have returned KERN_SUCCESS, return
* KERN_INVALID_ADDRESS instead.
*
* The addressability of the range is limited to that which can
* be described by a vm_address_t.
*
* RETURNS
* KERN_INVALID_TASK Bad task parameter
* KERN_INVALID_ARGUMENT both sync and async were specified.
* KERN_SUCCESS The usual.
* KERN_INVALID_ADDRESS There was a hole in the region.
*/
kern_return_t
vm_msync(
vm_map_t map,
vm_address_t address,
vm_size_t size,
vm_sync_t sync_flags)
{
if (map == VM_MAP_NULL) {
return KERN_INVALID_TASK;
}
return vm_map_msync(map, (vm_map_address_t)address,
(vm_map_size_t)size, sync_flags);
}
int
vm_toggle_entry_reuse(int toggle, int *old_value)
{
vm_map_t map = current_map();
assert(!map->is_nested_map);
if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) {
*old_value = map->disable_vmentry_reuse;
} else if (toggle == VM_TOGGLE_SET) {
vm_map_entry_t map_to_entry;
vm_map_lock(map);
vm_map_disable_hole_optimization(map);
map->disable_vmentry_reuse = TRUE;
__IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
if (map->first_free == map_to_entry) {
map->highest_entry_end = vm_map_min(map);
} else {
map->highest_entry_end = map->first_free->vme_end;
}
vm_map_unlock(map);
} else if (toggle == VM_TOGGLE_CLEAR) {
vm_map_lock(map);
map->disable_vmentry_reuse = FALSE;
vm_map_unlock(map);
} else {
return KERN_INVALID_ARGUMENT;
}
return KERN_SUCCESS;
}
/*
* mach_vm_behavior_set
*
* Sets the paging behavior attribute for the specified range
* in the specified map.
*
* This routine will fail with KERN_INVALID_ADDRESS if any address
* in [start,start+size) is not a valid allocated memory region.
*/
kern_return_t
mach_vm_behavior_set(
vm_map_t map,
mach_vm_offset_t start,
mach_vm_size_t size,
vm_behavior_t new_behavior)
{
vm_map_offset_t align_mask;
if ((map == VM_MAP_NULL) || (start + size < start)) {
return KERN_INVALID_ARGUMENT;
}
if (size == 0) {
return KERN_SUCCESS;
}
switch (new_behavior) {
case VM_BEHAVIOR_REUSABLE:
case VM_BEHAVIOR_REUSE:
case VM_BEHAVIOR_CAN_REUSE:
case VM_BEHAVIOR_ZERO:
/*
* Align to the hardware page size, to allow
* malloc() to maximize the amount of re-usability,
* even on systems with larger software page size.
*/
align_mask = PAGE_MASK;
break;
default:
align_mask = VM_MAP_PAGE_MASK(map);
break;
}
return vm_map_behavior_set(map,
vm_map_trunc_page(start, align_mask),
vm_map_round_page(start + size, align_mask),
new_behavior);
}
/*
* vm_behavior_set
*
* Sets the paging behavior attribute for the specified range
* in the specified map.
*
* This routine will fail with KERN_INVALID_ADDRESS if any address
* in [start,start+size) is not a valid allocated memory region.
*
* This routine is potentially limited in addressibility by the
* use of vm_offset_t (if the map provided is larger than the
* kernel's).
*/
kern_return_t
vm_behavior_set(
vm_map_t map,
vm_offset_t start,
vm_size_t size,
vm_behavior_t new_behavior)
{
if (start + size < start) {
return KERN_INVALID_ARGUMENT;
}
return mach_vm_behavior_set(map,
(mach_vm_offset_t) start,
(mach_vm_size_t) size,
new_behavior);
}
/*
* mach_vm_region:
*
* User call to obtain information about a region in
* a task's address map. Currently, only one flavor is
* supported.
*
* XXX The reserved and behavior fields cannot be filled
* in until the vm merge from the IK is completed, and
* vm_reserve is implemented.
*
* XXX Dependency: syscall_vm_region() also supports only one flavor.
*/
kern_return_t
mach_vm_region(
vm_map_t map,
mach_vm_offset_t *address, /* IN/OUT */
mach_vm_size_t *size, /* OUT */
vm_region_flavor_t flavor, /* IN */
vm_region_info_t info, /* OUT */
mach_msg_type_number_t *count, /* IN/OUT */
mach_port_t *object_name) /* OUT */
{
vm_map_offset_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_offset_t)*address;
map_size = (vm_map_size_t)*size;
/* legacy conversion */
if (VM_REGION_BASIC_INFO == flavor) {
flavor = VM_REGION_BASIC_INFO_64;
}
kr = vm_map_region(map,
&map_addr, &map_size,
flavor, info, count,
object_name);
*address = map_addr;
*size = map_size;
return kr;
}
/*
* vm_region_64 and vm_region:
*
* User call to obtain information about a region in
* a task's address map. Currently, only one flavor is
* supported.
*
* XXX The reserved and behavior fields cannot be filled
* in until the vm merge from the IK is completed, and
* vm_reserve is implemented.
*
* XXX Dependency: syscall_vm_region() also supports only one flavor.
*/
kern_return_t
vm_region_64(
vm_map_t map,
vm_offset_t *address, /* IN/OUT */
vm_size_t *size, /* OUT */
vm_region_flavor_t flavor, /* IN */
vm_region_info_t info, /* OUT */
mach_msg_type_number_t *count, /* IN/OUT */
mach_port_t *object_name) /* OUT */
{
vm_map_offset_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_offset_t)*address;
map_size = (vm_map_size_t)*size;
/* legacy conversion */
if (VM_REGION_BASIC_INFO == flavor) {
flavor = VM_REGION_BASIC_INFO_64;
}
kr = vm_map_region(map,
&map_addr, &map_size,
flavor, info, count,
object_name);
*address = CAST_DOWN(vm_offset_t, map_addr);
*size = CAST_DOWN(vm_size_t, map_size);
if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
return KERN_INVALID_ADDRESS;
}
return kr;
}
kern_return_t
vm_region(
vm_map_t map,
vm_address_t *address, /* IN/OUT */
vm_size_t *size, /* OUT */
vm_region_flavor_t flavor, /* IN */
vm_region_info_t info, /* OUT */
mach_msg_type_number_t *count, /* IN/OUT */
mach_port_t *object_name) /* OUT */
{
vm_map_address_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_address_t)*address;
map_size = (vm_map_size_t)*size;
kr = vm_map_region(map,
&map_addr, &map_size,
flavor, info, count,
object_name);
*address = CAST_DOWN(vm_address_t, map_addr);
*size = CAST_DOWN(vm_size_t, map_size);
if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
return KERN_INVALID_ADDRESS;
}
return kr;
}
/*
* vm_region_recurse: A form of vm_region which follows the
* submaps in a target map
*
*/
kern_return_t
mach_vm_region_recurse(
vm_map_t map,
mach_vm_address_t *address,
mach_vm_size_t *size,
uint32_t *depth,
vm_region_recurse_info_t info,
mach_msg_type_number_t *infoCnt)
{
vm_map_address_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_address_t)*address;
map_size = (vm_map_size_t)*size;
kr = vm_map_region_recurse_64(
map,
&map_addr,
&map_size,
depth,
(vm_region_submap_info_64_t)info,
infoCnt);
*address = map_addr;
*size = map_size;
return kr;
}
/*
* vm_region_recurse: A form of vm_region which follows the
* submaps in a target map
*
*/
kern_return_t
vm_region_recurse_64(
vm_map_t map,
vm_address_t *address,
vm_size_t *size,
uint32_t *depth,
vm_region_recurse_info_64_t info,
mach_msg_type_number_t *infoCnt)
{
vm_map_address_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_address_t)*address;
map_size = (vm_map_size_t)*size;
kr = vm_map_region_recurse_64(
map,
&map_addr,
&map_size,
depth,
(vm_region_submap_info_64_t)info,
infoCnt);
*address = CAST_DOWN(vm_address_t, map_addr);
*size = CAST_DOWN(vm_size_t, map_size);
if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
return KERN_INVALID_ADDRESS;
}
return kr;
}
kern_return_t
vm_region_recurse(
vm_map_t map,
vm_offset_t *address, /* IN/OUT */
vm_size_t *size, /* OUT */
natural_t *depth, /* IN/OUT */
vm_region_recurse_info_t info32, /* IN/OUT */
mach_msg_type_number_t *infoCnt) /* IN/OUT */
{
vm_region_submap_info_data_64_t info64;
vm_region_submap_info_t info;
vm_map_address_t map_addr;
vm_map_size_t map_size;
kern_return_t kr;
if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) {
return KERN_INVALID_ARGUMENT;
}
map_addr = (vm_map_address_t)*address;
map_size = (vm_map_size_t)*size;
info = (vm_region_submap_info_t)info32;
*infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
kr = vm_map_region_recurse_64(map, &map_addr, &map_size,
depth, &info64, infoCnt);
info->protection = info64.protection;
info->max_protection = info64.max_protection;
info->inheritance = info64.inheritance;
info->offset = (uint32_t)info64.offset; /* trouble-maker */
info->user_tag = info64.user_tag;
info->pages_resident = info64.pages_resident;
info->pages_shared_now_private = info64.pages_shared_now_private;
info->pages_swapped_out = info64.pages_swapped_out;
info->pages_dirtied = info64.pages_dirtied;
info->ref_count = info64.ref_count;
info->shadow_depth = info64.shadow_depth;
info->external_pager = info64.external_pager;
info->share_mode = info64.share_mode;
info->is_submap = info64.is_submap;
info->behavior = info64.behavior;
info->object_id = info64.object_id;
info->user_wired_count = info64.user_wired_count;
*address = CAST_DOWN(vm_address_t, map_addr);
*size = CAST_DOWN(vm_size_t, map_size);
*infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
if (KERN_SUCCESS == kr && map_addr + map_size > VM_MAX_ADDRESS) {
return KERN_INVALID_ADDRESS;
}
return kr;
}
kern_return_t
mach_vm_purgable_control(
vm_map_t map,
mach_vm_offset_t address,
vm_purgable_t control,
int *state)
{
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
/* not allowed from user-space */
return KERN_INVALID_ARGUMENT;
}
return vm_map_purgable_control(map,
vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
control,
state);
}
kern_return_t
mach_vm_purgable_control_external(
mach_port_t target_tport,
mach_vm_offset_t address,
vm_purgable_t control,
int *state)
{
vm_map_t map;
kern_return_t kr;
if (control == VM_PURGABLE_GET_STATE) {
map = convert_port_to_map_read(target_tport);
} else {
map = convert_port_to_map(target_tport);
}
kr = mach_vm_purgable_control(map, address, control, state);
vm_map_deallocate(map);
return kr;
}
kern_return_t
vm_purgable_control(
vm_map_t map,
vm_offset_t address,
vm_purgable_t control,
int *state)
{
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
if (control == VM_PURGABLE_SET_STATE_FROM_KERNEL) {
/* not allowed from user-space */
return KERN_INVALID_ARGUMENT;
}
return vm_map_purgable_control(map,
vm_map_trunc_page(address, VM_MAP_PAGE_MASK(map)),
control,
state);
}
kern_return_t
vm_purgable_control_external(
mach_port_t target_tport,
vm_offset_t address,
vm_purgable_t control,
int *state)
{
vm_map_t map;
kern_return_t kr;
if (control == VM_PURGABLE_GET_STATE) {
map = convert_port_to_map_read(target_tport);
} else {
map = convert_port_to_map(target_tport);
}
kr = vm_purgable_control(map, address, control, state);
vm_map_deallocate(map);
return kr;
}
kern_return_t
mach_vm_page_query(
vm_map_t map,
mach_vm_offset_t offset,
int *disposition,
int *ref_count)
{
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
return vm_map_page_query_internal(
map,
vm_map_trunc_page(offset, PAGE_MASK),
disposition, ref_count);
}
kern_return_t
vm_map_page_query(
vm_map_t map,
vm_offset_t offset,
int *disposition,
int *ref_count)
{
if (VM_MAP_NULL == map) {
return KERN_INVALID_ARGUMENT;
}
return vm_map_page_query_internal(
map,
vm_map_trunc_page(offset, PAGE_MASK),
disposition, ref_count);
}
kern_return_t
mach_vm_page_range_query(
vm_map_t map,
mach_vm_offset_t address,
mach_vm_size_t size,
mach_vm_address_t dispositions_addr,
mach_vm_size_t *dispositions_count)
{
kern_return_t kr = KERN_SUCCESS;
int num_pages = 0, i = 0;
mach_vm_size_t curr_sz = 0, copy_sz = 0;
mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
mach_msg_type_number_t count = 0;
void *info = NULL;
void *local_disp = NULL;
vm_map_size_t info_size = 0, local_disp_size = 0;
mach_vm_offset_t start = 0, end = 0;
int effective_page_shift, effective_page_size, effective_page_mask;
if (map == VM_MAP_NULL || dispositions_count == NULL) {
return KERN_INVALID_ARGUMENT;
}
effective_page_shift = vm_self_region_page_shift_safely(map);
if (effective_page_shift == -1) {
return KERN_INVALID_ARGUMENT;
}
effective_page_size = (1 << effective_page_shift);
effective_page_mask = effective_page_size - 1;
if (os_mul_overflow(*dispositions_count, sizeof(int), &disp_buf_req_size)) {
return KERN_INVALID_ARGUMENT;
}
start = vm_map_trunc_page(address, effective_page_mask);
end = vm_map_round_page(address + size, effective_page_mask);
if (end < start) {
return KERN_INVALID_ARGUMENT;
}
if ((end - start) < size) {
/*
* Aligned size is less than unaligned size.
*/
return KERN_INVALID_ARGUMENT;
}
if (disp_buf_req_size == 0 || (end == start)) {
return KERN_SUCCESS;
}
/*
* For large requests, we will go through them
* MAX_PAGE_RANGE_QUERY chunk at a time.
*/
curr_sz = MIN(end - start, MAX_PAGE_RANGE_QUERY);
num_pages = (int) (curr_sz >> effective_page_shift);
info_size = num_pages * sizeof(vm_page_info_basic_data_t);
info = kalloc_data(info_size, Z_WAITOK);
local_disp_size = num_pages * sizeof(int);
local_disp = kalloc_data(local_disp_size, Z_WAITOK);
if (info == NULL || local_disp == NULL) {
kr = KERN_RESOURCE_SHORTAGE;
goto out;
}
while (size) {
count = VM_PAGE_INFO_BASIC_COUNT;
kr = vm_map_page_range_info_internal(
map,
start,
vm_map_round_page(start + curr_sz, effective_page_mask),
effective_page_shift,
VM_PAGE_INFO_BASIC,
(vm_page_info_t) info,
&count);
assert(kr == KERN_SUCCESS);
for (i = 0; i < num_pages; i++) {
((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
}
copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */);
kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
start += curr_sz;
disp_buf_req_size -= copy_sz;
disp_buf_total_size += copy_sz;
if (kr != 0) {
break;
}
if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
/*
* We might have inspected the full range OR
* more than it esp. if the user passed in
* non-page aligned start/size and/or if we
* descended into a submap. We are done here.
*/
size = 0;
} else {
dispositions_addr += copy_sz;
size -= curr_sz;
curr_sz = MIN(vm_map_round_page(size, effective_page_mask), MAX_PAGE_RANGE_QUERY);
num_pages = (int)(curr_sz >> effective_page_shift);
}
}
*dispositions_count = disp_buf_total_size / sizeof(int);
out:
kfree_data(local_disp, local_disp_size);
kfree_data(info, info_size);
return kr;
}
kern_return_t
mach_vm_page_info(
vm_map_t map,
mach_vm_address_t address,
vm_page_info_flavor_t flavor,
vm_page_info_t info,
mach_msg_type_number_t *count)
{
kern_return_t kr;
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
kr = vm_map_page_info(map, address, flavor, info, count);
return kr;
}
/*
* task_wire
*
* Set or clear the map's wiring_required flag. This flag, if set,
* will cause all future virtual memory allocation to allocate
* user wired memory. Unwiring pages wired down as a result of
* this routine is done with the vm_wire interface.
*/
kern_return_t
task_wire(
vm_map_t map,
boolean_t must_wire __unused)
{
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
return KERN_NOT_SUPPORTED;
}
kern_return_t
vm_map_exec_lockdown(
vm_map_t map)
{
if (map == VM_MAP_NULL) {
return KERN_INVALID_ARGUMENT;
}
vm_map_lock(map);
map->map_disallow_new_exec = TRUE;
vm_map_unlock(map);
return KERN_SUCCESS;
}
#if XNU_PLATFORM_MacOSX
/*
* Now a kernel-private interface (for BootCache
* use only). Need a cleaner way to create an
* empty vm_map() and return a handle to it.
*/
kern_return_t
vm_region_object_create(
vm_map_t target_map,
vm_size_t size,
ipc_port_t *object_handle)
{
vm_named_entry_t user_entry;
vm_map_t new_map;
user_entry = mach_memory_entry_allocate(object_handle);
/* Create a named object based on a submap of specified size */
new_map = vm_map_create_options(PMAP_NULL, VM_MAP_MIN_ADDRESS,
vm_map_round_page(size, VM_MAP_PAGE_MASK(target_map)),
VM_MAP_CREATE_PAGEABLE);
vm_map_set_page_shift(new_map, VM_MAP_PAGE_SHIFT(target_map));
user_entry->backing.map = new_map;
user_entry->internal = TRUE;
user_entry->is_sub_map = TRUE;
user_entry->offset = 0;
user_entry->protection = VM_PROT_ALL;
user_entry->size = size;
return KERN_SUCCESS;
}
#endif /* XNU_PLATFORM_MacOSX */
kern_return_t
mach_vm_deferred_reclamation_buffer_init(
task_t task,
mach_vm_offset_t *address,
mach_vm_size_t size)
{
#if CONFIG_DEFERRED_RECLAIM
return vm_deferred_reclamation_buffer_init_internal(task, address, size);
#else
(void) task;
(void) address;
(void) size;
(void) indices;
return KERN_NOT_SUPPORTED;
#endif /* CONFIG_DEFERRED_RECLAIM */
}
kern_return_t
mach_vm_deferred_reclamation_buffer_synchronize(
task_t task,
mach_vm_size_t num_entries_to_reclaim)
{
#if CONFIG_DEFERRED_RECLAIM
return vm_deferred_reclamation_buffer_synchronize_internal(task, num_entries_to_reclaim);
#else
(void) task;
(void) num_entries_to_reclaim;
return KERN_NOT_SUPPORTED;
#endif /* CONFIG_DEFERRED_RECLAIM */
}
kern_return_t
mach_vm_deferred_reclamation_buffer_update_reclaimable_bytes(task_t task, mach_vm_size_t reclaimable_bytes)
{
#if CONFIG_DEFERRED_RECLAIM
return vm_deferred_reclamation_buffer_update_reclaimable_bytes_internal(task, reclaimable_bytes);
#else
(void) task;
(void) reclaimable_bytes;
return KERN_NOT_SUPPORTED;
#endif /* CONFIG_DEFERRED_RECLAIM */
}
#if CONFIG_MAP_RANGES
extern void qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
static int
vm_map_user_range_cmp(const void *e1, const void *e2)
{
const struct vm_map_user_range *r1 = e1;
const struct vm_map_user_range *r2 = e2;
if (r1->vmur_min_address != r2->vmur_min_address) {
return r1->vmur_min_address < r2->vmur_min_address ? -1 : 1;
}
return 0;
}
static int
mach_vm_range_recipe_v1_cmp(const void *e1, const void *e2)
{
const mach_vm_range_recipe_v1_t *r1 = e1;
const mach_vm_range_recipe_v1_t *r2 = e2;
if (r1->range.min_address != r2->range.min_address) {
return r1->range.min_address < r2->range.min_address ? -1 : 1;
}
return 0;
}
/*!
* @function mach_vm_range_create_v1()
*
* @brief
* Handle the backend for mach_vm_range_create() for the
* MACH_VM_RANGE_FLAVOR_V1 flavor.
*
* @description
* This call allows to create "ranges" in the map of a task
* that have special semantics/policies around placement of
* new allocations (in the vm_map_locate_space() sense).
*
* @returns
* - KERN_SUCCESS on success
* - KERN_INVALID_ARGUMENT for incorrect arguments
* - KERN_NO_SPACE if the maximum amount of ranges would be exceeded
* - KERN_MEMORY_PRESENT if any of the requested ranges
* overlaps with existing ranges or allocations in the map.
*/
static kern_return_t
mach_vm_range_create_v1(
vm_map_t map,
mach_vm_range_recipe_v1_t *recipe,
uint32_t new_count)
{
const vm_offset_t mask = VM_MAP_PAGE_MASK(map);
vm_map_user_range_t table;
kern_return_t kr = KERN_SUCCESS;
uint16_t count;
struct mach_vm_range void1 = {
.min_address = map->default_range.max_address,
.max_address = map->data_range.min_address,
};
struct mach_vm_range void2 = {
.min_address = map->data_range.max_address,
#if XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT
.max_address = MACH_VM_JUMBO_ADDRESS,
#else /* !XNU_TARGET_OS_IOS || !EXTENDED_USER_VA_SUPPORT */
.max_address = vm_map_max(map),
#endif /* XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT */
};
qsort(recipe, new_count, sizeof(mach_vm_range_recipe_v1_t),
mach_vm_range_recipe_v1_cmp);
/*
* Step 1: Validate that the recipes have no intersections.
*/
for (size_t i = 0; i < new_count; i++) {
mach_vm_range_t r = &recipe[i].range;
mach_vm_size_t s;
if (recipe[i].flags) {
return KERN_INVALID_ARGUMENT;
}
static_assert(UMEM_RANGE_ID_FIXED == MACH_VM_RANGE_FIXED);
switch (recipe[i].range_tag) {
case MACH_VM_RANGE_FIXED:
break;
default:
return KERN_INVALID_ARGUMENT;
}
if (!VM_MAP_PAGE_ALIGNED(r->min_address, mask) ||
!VM_MAP_PAGE_ALIGNED(r->max_address, mask) ||
r->min_address >= r->max_address) {
return KERN_INVALID_ARGUMENT;
}
s = mach_vm_range_size(r);
if (!mach_vm_range_contains(&void1, r->min_address, s) &&
!mach_vm_range_contains(&void2, r->min_address, s)) {
return KERN_INVALID_ARGUMENT;
}
if (i > 0 && recipe[i - 1].range.max_address >
recipe[i].range.min_address) {
return KERN_INVALID_ARGUMENT;
}
}
vm_map_lock(map);
table = map->extra_ranges;
count = map->extra_ranges_count;
if (count + new_count > VM_MAP_EXTRA_RANGES_MAX) {
kr = KERN_NO_SPACE;
goto out_unlock;
}
/*
* Step 2: Check that there is no intersection with existing ranges.
*/
for (size_t i = 0, j = 0; i < new_count && j < count;) {
mach_vm_range_t r1 = &recipe[i].range;
vm_map_user_range_t r2 = &table[j];
if (r1->max_address <= r2->vmur_min_address) {
i++;
} else if (r2->vmur_max_address <= r1->min_address) {
j++;
} else {
kr = KERN_MEMORY_PRESENT;
goto out_unlock;
}
}
/*
* Step 4: commit the new ranges.
*/
static_assert(VM_MAP_EXTRA_RANGES_MAX * sizeof(struct vm_map_user_range) <=
KALLOC_SAFE_ALLOC_SIZE);
table = krealloc_data(table,
count * sizeof(struct vm_map_user_range),
(count + new_count) * sizeof(struct vm_map_user_range),
Z_ZERO | Z_WAITOK | Z_NOFAIL);
for (size_t i = 0; i < new_count; i++) {
static_assert(MACH_VM_MAX_ADDRESS < (1ull << 56));
table[count + i] = (struct vm_map_user_range){
.vmur_min_address = recipe[i].range.min_address,
.vmur_max_address = recipe[i].range.max_address,
.vmur_range_id = (vm_map_range_id_t)recipe[i].range_tag,
};
}
qsort(table, count + new_count,
sizeof(struct vm_map_user_range), vm_map_user_range_cmp);
map->extra_ranges_count += new_count;
map->extra_ranges = table;
out_unlock:
vm_map_unlock(map);
if (kr == KERN_SUCCESS) {
for (size_t i = 0; i < new_count; i++) {
vm_map_kernel_flags_t vmk_flags = {
.vmf_fixed = true,
.vmf_overwrite = true,
.vmkf_overwrite_immutable = true,
.vm_tag = recipe[i].vm_tag,
};
__assert_only kern_return_t kr2;
kr2 = vm_map_enter(map, &recipe[i].range.min_address,
mach_vm_range_size(&recipe[i].range),
0, vmk_flags, VM_OBJECT_NULL, 0, FALSE,
VM_PROT_NONE, VM_PROT_ALL,
VM_INHERIT_DEFAULT);
assert(kr2 == KERN_SUCCESS);
}
}
return kr;
}
kern_return_t
mach_vm_range_create(
vm_map_t map,
mach_vm_range_flavor_t flavor,
mach_vm_range_recipes_raw_t recipe,
natural_t size)
{
if (map != current_map()) {
return KERN_INVALID_ARGUMENT;
}
if (!map->uses_user_ranges) {
return KERN_NOT_SUPPORTED;
}
if (size == 0) {
return KERN_SUCCESS;
}
if (flavor == MACH_VM_RANGE_FLAVOR_V1) {
mach_vm_range_recipe_v1_t *array;
if (size % sizeof(mach_vm_range_recipe_v1_t)) {
return KERN_INVALID_ARGUMENT;
}
size /= sizeof(mach_vm_range_recipe_v1_t);
if (size > VM_MAP_EXTRA_RANGES_MAX) {
return KERN_NO_SPACE;
}
array = (mach_vm_range_recipe_v1_t *)recipe;
return mach_vm_range_create_v1(map, array, size);
}
return KERN_INVALID_ARGUMENT;
}
#else /* !CONFIG_MAP_RANGES */
kern_return_t
mach_vm_range_create(
vm_map_t map,
mach_vm_range_flavor_t flavor,
mach_vm_range_recipes_raw_t recipe,
natural_t size)
{
#pragma unused(map, flavor, recipe, size)
return KERN_NOT_SUPPORTED;
}
#endif /* !CONFIG_MAP_RANGES */
/*
* These symbols are looked up at runtime by vmware, VirtualBox,
* despite not being exported in the symbol sets.
*/
#if defined(__x86_64__)
extern typeof(mach_vm_remap_external) mach_vm_remap;
extern typeof(mach_vm_map_external) mach_vm_map;
extern typeof(vm_map_external) vm_map;
kern_return_t
mach_vm_map(
vm_map_t target_map,
mach_vm_offset_ut *address,
mach_vm_size_ut initial_size,
mach_vm_offset_ut mask,
int flags,
ipc_port_t port,
memory_object_offset_ut offset,
boolean_t copy,
vm_prot_ut cur_protection,
vm_prot_ut max_protection,
vm_inherit_ut inheritance)
{
return mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
offset, copy, cur_protection, max_protection, inheritance);
}
kern_return_t
mach_vm_remap(
vm_map_t target_map,
mach_vm_offset_ut *address,
mach_vm_size_ut size,
mach_vm_offset_ut mask,
int flags,
vm_map_t src_map,
mach_vm_offset_ut memory_address,
boolean_t copy,
vm_prot_ut *cur_protection, /* OUT */
vm_prot_ut *max_protection, /* OUT */
vm_inherit_ut inheritance)
{
return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
copy, cur_protection, max_protection, inheritance);
}
kern_return_t
vm_map(
vm_map_t target_map,
vm_offset_ut *address,
vm_size_ut size,
vm_offset_ut mask,
int flags,
ipc_port_t port,
vm_offset_ut offset,
boolean_t copy,
vm_prot_ut cur_protection,
vm_prot_ut max_protection,
vm_inherit_ut inheritance)
{
return mach_vm_map(target_map, address,
size, mask, flags, port, offset, copy,
cur_protection, max_protection, inheritance);
}
#endif /* __x86_64__ */