/*
* Copyright (c) 2000-2020 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
/*
* Mach Operating System
* Copyright (c) 1991,1990,1989 Carnegie Mellon University
* All Rights Reserved.
*
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
*
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
*
* Carnegie Mellon requests users of this software to return to
*
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
*
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
/*
* NOTICE: This file was modified by McAfee Research in 2004 to introduce
* support for mandatory and extensible security protections. This notice
* is included in support of clause 2.2 (b) of the Apple Public License,
* Version 2.0.
* Copyright (c) 2005-2006 SPARTA, Inc.
*/
/*
*/
/*
* File: ipc/ipc_object.c
* Author: Rich Draves
* Date: 1989
*
* Functions to manipulate IPC objects.
*/
#include <mach/mach_types.h>
#include <mach/boolean.h>
#include <mach/kern_return.h>
#include <mach/port.h>
#include <mach/message.h>
#include <kern/kern_types.h>
#include <kern/misc_protos.h>
#include <kern/ipc_kobject.h>
#include <kern/zalloc_internal.h> // zone_id_for_element
#include <ipc/ipc_types.h>
#include <ipc/ipc_importance.h>
#include <ipc/port.h>
#include <ipc/ipc_space.h>
#include <ipc/ipc_entry.h>
#include <ipc/ipc_object.h>
#include <ipc/ipc_hash.h>
#include <ipc/ipc_kmsg.h>
#include <ipc/ipc_right.h>
#include <ipc/ipc_notify.h>
#include <ipc/ipc_port.h>
#include <ipc/ipc_pset.h>
#include <security/mac_mach_internal.h>
static struct mpsc_daemon_queue ipc_object_deallocate_queue;
SECURITY_READ_ONLY_LATE(zone_t) ipc_object_zones[IOT_NUMBER];
/*
* In order to do lockfree lookups in the IPC space, we combine two schemes:
*
* - the ipc table pointer is protected with hazard pointers to allow
* dereferencing it with only holding a ref on a task or space;
*
* - we use ipc_object_lock_allow_invalid in order to lock locks and validate
* that they are the droid we're looking for.
*
* The second half requires that virtual addresses assigned that ever held
* a port, either hold a port, or nothing, forever. To get this property,
* we just piggy back on the zone sequestering security feature which gives
* us exactly that.
*
* However, sequestering really only "works" on a sufficiently large address
* space, especially for a resource that can be made by userspace at will,
* so we can't do lockless lookups on ILP32.
*
* Note: this scheme is incompatible with kasan quarantines
* (because it uses elements to store backtraces in them
* which lets the waitq lock appear "valid" by accident when
* elements are freed).
*/
#define IPC_OBJECT_ZC_BASE (ZC_ZFREE_CLEARMEM | ZC_SEQUESTER)
ZONE_INIT(&ipc_object_zones[IOT_PORT],
"ipc ports", sizeof(struct ipc_port),
IPC_OBJECT_ZC_BASE | ZC_CACHING, ZONE_ID_IPC_PORT, NULL);
ZONE_INIT(&ipc_object_zones[IOT_PORT_SET],
"ipc port sets", sizeof(struct ipc_pset),
IPC_OBJECT_ZC_BASE, ZONE_ID_IPC_PORT_SET, NULL);
__attribute__((noinline))
static void
ipc_object_free(unsigned int otype, ipc_object_t object, bool last_ref)
{
if (last_ref) {
if (otype == IOT_PORT) {
ipc_port_finalize(ip_object_to_port(object));
} else {
ipc_pset_finalize(ips_object_to_pset(object));
}
}
zfree(ipc_object_zones[otype], object);
}
__attribute__((noinline))
static void
ipc_object_free_safe(ipc_object_t object)
{
struct waitq *wq = io_waitq(object);
assert(!waitq_is_valid(wq));
assert(os_atomic_load(&wq->waitq_defer.mpqc_next, relaxed) == NULL);
mpsc_daemon_enqueue(&ipc_object_deallocate_queue,
&wq->waitq_defer, MPSC_QUEUE_NONE);
}
static void
ipc_object_deallocate_queue_invoke(mpsc_queue_chain_t e,
__assert_only mpsc_daemon_queue_t dq)
{
struct waitq *wq = __container_of(e, struct waitq, waitq_defer);
ipc_object_t io = io_from_waitq(wq);
assert(dq == &ipc_object_deallocate_queue);
os_atomic_store(&wq->waitq_defer.mpqc_next, NULL, relaxed);
ipc_object_free(io_otype(io), io, true);
}
void
ipc_object_deallocate_register_queue(void)
{
thread_deallocate_daemon_register_queue(&ipc_object_deallocate_queue,
ipc_object_deallocate_queue_invoke);
}
/*
* Routine: ipc_object_reference
* Purpose:
* Take a reference to an object.
*/
void
ipc_object_reference(
ipc_object_t io)
{
static_assert(sizeof(os_ref_atomic_t) == sizeof(io->io_references));
os_ref_retain_raw((os_ref_atomic_t *)&io->io_references, NULL);
}
/*
* Routine: ipc_object_release
* Purpose:
* Release a reference to an object.
*/
void
ipc_object_release(
ipc_object_t io)
{
#if DEBUG
assert(get_preemption_level() == 0);
#endif
if (os_ref_release_raw((os_ref_atomic_t *)&io->io_references, NULL) == 0) {
/* Free the object */
ipc_object_free(io_otype(io), io, true);
}
}
/*
* Routine: ipc_object_release_safe
* Purpose:
* Release a reference to an object safely
*/
void
ipc_object_release_safe(
ipc_object_t io)
{
if (os_ref_release_raw((os_ref_atomic_t *)&io->io_references, NULL) == 0) {
if (get_preemption_level() == 0) {
ipc_object_free(io_otype(io), io, true);
} else {
ipc_object_free_safe(io);
}
}
}
/*
* Routine: ipc_object_release_live
* Purpose:
* Release a reference to an object that isn't the last one.
*/
void
ipc_object_release_live(
ipc_object_t io)
{
os_ref_release_live_raw((os_ref_atomic_t *)&io->io_references, NULL);
}
/*
* Routine: ipc_object_translate
* Purpose:
* Look up an object in a space.
* Conditions:
* Nothing locked before. If successful, the object
* is returned active and locked. The caller doesn't get a ref.
* Returns:
* KERN_SUCCESS Object returned locked.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_NAME The name doesn't denote a right
* KERN_INVALID_RIGHT Name doesn't denote the correct right
*/
kern_return_t
ipc_object_translate(
ipc_space_t space,
mach_port_name_t name,
mach_port_right_t right,
ipc_object_t *objectp)
{
ipc_entry_bits_t bits;
ipc_object_t object;
kern_return_t kr;
if (!MACH_PORT_RIGHT_VALID_TRANSLATE(right)) {
return KERN_INVALID_RIGHT;
}
kr = ipc_right_lookup_read(space, name, &bits, &object);
if (kr != KERN_SUCCESS) {
return kr;
}
/* object is locked and active */
if ((bits & MACH_PORT_TYPE(right)) == MACH_PORT_TYPE_NONE) {
io_unlock(object);
return KERN_INVALID_RIGHT;
}
*objectp = object;
return KERN_SUCCESS;
}
/*
* Routine: ipc_object_translate_two
* Purpose:
* Look up two objects in a space.
* Conditions:
* Nothing locked before. If successful, the objects
* are returned locked. The caller doesn't get a ref.
* Returns:
* KERN_SUCCESS Objects returned locked.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_NAME A name doesn't denote a right.
* KERN_INVALID_RIGHT A name doesn't denote the correct right.
*/
kern_return_t
ipc_object_translate_two(
ipc_space_t space,
mach_port_name_t name1,
mach_port_right_t right1,
ipc_object_t *objectp1,
mach_port_name_t name2,
mach_port_right_t right2,
ipc_object_t *objectp2)
{
ipc_entry_t entry1;
ipc_entry_t entry2;
ipc_object_t object1, object2;
kern_return_t kr;
boolean_t doguard = TRUE;
kr = ipc_right_lookup_two_read(space, name1, &entry1, name2, &entry2);
if (kr != KERN_SUCCESS) {
return kr;
}
/* space is read-locked and active */
if ((entry1->ie_bits & MACH_PORT_TYPE(right1)) == MACH_PORT_TYPE_NONE) {
/* If looking for receive, and the entry used to hold one, give a pass on EXC_GUARD */
if ((right1 & MACH_PORT_RIGHT_RECEIVE) == MACH_PORT_RIGHT_RECEIVE &&
(entry1->ie_bits & MACH_PORT_TYPE_EX_RECEIVE) == MACH_PORT_TYPE_EX_RECEIVE) {
doguard = FALSE;
}
is_read_unlock(space);
if (doguard) {
mach_port_guard_exception(name1, 0, 0, kGUARD_EXC_INVALID_RIGHT);
}
return KERN_INVALID_RIGHT;
}
if ((entry2->ie_bits & MACH_PORT_TYPE(right2)) == MACH_PORT_TYPE_NONE) {
/* If looking for receive, and the entry used to hold one, give a pass on EXC_GUARD */
if ((right2 & MACH_PORT_RIGHT_RECEIVE) == MACH_PORT_RIGHT_RECEIVE &&
(entry2->ie_bits & MACH_PORT_TYPE_EX_RECEIVE) == MACH_PORT_TYPE_EX_RECEIVE) {
doguard = FALSE;
}
is_read_unlock(space);
if (doguard) {
mach_port_guard_exception(name2, 0, 0, kGUARD_EXC_INVALID_RIGHT);
}
return KERN_INVALID_RIGHT;
}
object1 = entry1->ie_object;
assert(object1 != IO_NULL);
io_lock(object1);
if (!io_active(object1)) {
io_unlock(object1);
is_read_unlock(space);
return KERN_INVALID_NAME;
}
object2 = entry2->ie_object;
assert(object2 != IO_NULL);
io_lock(object2);
if (!io_active(object2)) {
io_unlock(object1);
io_unlock(object2);
is_read_unlock(space);
return KERN_INVALID_NAME;
}
*objectp1 = object1;
*objectp2 = object2;
is_read_unlock(space);
return KERN_SUCCESS;
}
/*
* Routine: ipc_object_alloc_dead
* Purpose:
* Allocate a dead-name entry.
* Conditions:
* Nothing locked.
* Returns:
* KERN_SUCCESS The dead name is allocated.
* KERN_INVALID_TASK The space is dead.
* KERN_NO_SPACE No room for an entry in the space.
*/
kern_return_t
ipc_object_alloc_dead(
ipc_space_t space,
mach_port_name_t *namep)
{
ipc_entry_t entry;
kern_return_t kr;
kr = ipc_entry_alloc(space, IO_NULL, namep, &entry);
if (kr != KERN_SUCCESS) {
return kr;
}
/* space is write-locked */
/* null object, MACH_PORT_TYPE_DEAD_NAME, 1 uref */
entry->ie_bits |= MACH_PORT_TYPE_DEAD_NAME | 1;
ipc_entry_modified(space, *namep, entry);
is_write_unlock(space);
return KERN_SUCCESS;
}
/*
* Routine: ipc_object_alloc
* Purpose:
* Allocate an object.
* Conditions:
* Nothing locked.
* The space is write locked on successful return.
* The caller doesn't get a reference for the object.
* Returns:
* KERN_SUCCESS The object is allocated.
* KERN_INVALID_TASK The space is dead.
* KERN_NO_SPACE No room for an entry in the space.
*/
kern_return_t
ipc_object_alloc(
ipc_space_t space,
ipc_object_type_t otype,
mach_port_type_t type,
mach_port_urefs_t urefs,
mach_port_name_t *namep,
ipc_object_t *objectp)
{
ipc_object_t object;
ipc_entry_t entry;
kern_return_t kr;
assert(otype < IOT_NUMBER);
assert((type & MACH_PORT_TYPE_ALL_RIGHTS) == type);
assert(type != MACH_PORT_TYPE_NONE);
assert(urefs <= MACH_PORT_UREFS_MAX);
object = io_alloc(otype, Z_WAITOK | Z_ZERO | Z_NOFAIL);
os_atomic_init(&object->io_bits, io_makebits(otype));
os_atomic_init(&object->io_references, 1); /* for entry, not caller */
*namep = CAST_MACH_PORT_TO_NAME(object);
kr = ipc_entry_alloc(space, object, namep, &entry);
if (kr != KERN_SUCCESS) {
ipc_object_free(otype, object, false);
return kr;
}
/* space is write-locked */
entry->ie_bits |= type | urefs;
ipc_entry_modified(space, *namep, entry);
*objectp = object;
return KERN_SUCCESS;
}
/*
* Routine: ipc_object_alloc_name
* Purpose:
* Allocate an object, with a specific name.
* Conditions:
* Nothing locked. If successful, the object is returned locked.
* The caller doesn't get a reference for the object.
*
* finish_init() must call an ipc_*_init function
* that will return the object locked (using IPC_PORT_INIT_LOCKED,
* or SYNC_POLICY_INIT_LOCKED, or equivalent).
*
* Returns:
* KERN_SUCCESS The object is allocated.
* KERN_INVALID_TASK The space is dead.
* KERN_NAME_EXISTS The name already denotes a right.
*/
kern_return_t
ipc_object_alloc_name(
ipc_space_t space,
ipc_object_type_t otype,
mach_port_type_t type,
mach_port_urefs_t urefs,
mach_port_name_t name,
ipc_object_t *objectp,
void (^finish_init)(ipc_object_t))
{
ipc_object_t object;
ipc_entry_t entry;
kern_return_t kr;
assert(otype < IOT_NUMBER);
assert((type & MACH_PORT_TYPE_ALL_RIGHTS) == type);
assert(type != MACH_PORT_TYPE_NONE);
assert(urefs <= MACH_PORT_UREFS_MAX);
object = io_alloc(otype, Z_WAITOK | Z_ZERO | Z_NOFAIL);
os_atomic_init(&object->io_bits, io_makebits(otype));
os_atomic_init(&object->io_references, 1); /* for entry, not caller */
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS) {
ipc_object_free(otype, object, false);
return kr;
}
/* space is write-locked */
if (ipc_right_inuse(entry)) {
is_write_unlock(space);
ipc_object_free(otype, object, false);
return KERN_NAME_EXISTS;
}
entry->ie_bits |= type | urefs;
entry->ie_object = object;
finish_init(object);
/* object is locked */
io_lock_held(object);
ipc_entry_modified(space, name, entry);
is_write_unlock(space);
*objectp = object;
return KERN_SUCCESS;
}
/* Routine: ipc_object_validate
* Purpose:
* Validates an ipc port or port set as belonging to the correct
* zone.
*/
void
ipc_object_validate(
ipc_object_t object,
ipc_object_type_t type)
{
if (type != IOT_PORT_SET) {
ip_validate(object);
} else {
ips_validate(object);
}
}
void
ipc_object_validate_aligned(
ipc_object_t object,
ipc_object_type_t type)
{
if (type != IOT_PORT_SET) {
ip_validate_aligned(object);
} else {
ips_validate_aligned(object);
}
}
/*
* Routine: ipc_object_copyin_type
* Purpose:
* Convert a send type name to a received type name.
*/
mach_msg_type_name_t
ipc_object_copyin_type(
mach_msg_type_name_t msgt_name)
{
switch (msgt_name) {
case MACH_MSG_TYPE_MOVE_RECEIVE:
return MACH_MSG_TYPE_PORT_RECEIVE;
case MACH_MSG_TYPE_MOVE_SEND_ONCE:
case MACH_MSG_TYPE_MAKE_SEND_ONCE:
return MACH_MSG_TYPE_PORT_SEND_ONCE;
case MACH_MSG_TYPE_MOVE_SEND:
case MACH_MSG_TYPE_MAKE_SEND:
case MACH_MSG_TYPE_COPY_SEND:
return MACH_MSG_TYPE_PORT_SEND;
default:
return MACH_MSG_TYPE_PORT_NONE;
}
}
/*
* Routine: ipc_object_copyin
* Purpose:
* Copyin a capability from a space.
* If successful, the caller gets a ref
* for the resulting object, unless it is IO_DEAD.
* Conditions:
* Nothing locked.
* Returns:
* KERN_SUCCESS Acquired an object, possibly IO_DEAD.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_NAME Name doesn't exist in space.
* KERN_INVALID_RIGHT Name doesn't denote correct right.
*/
kern_return_t
ipc_object_copyin(
ipc_space_t space,
mach_port_name_t name,
mach_msg_type_name_t msgt_name,
ipc_object_t *objectp,
mach_port_context_t context,
mach_msg_guard_flags_t *guard_flags,
ipc_object_copyin_flags_t copyin_flags)
{
ipc_entry_t entry;
ipc_port_t soright;
ipc_port_t release_port;
kern_return_t kr;
int assertcnt = 0;
ipc_object_copyin_flags_t copyin_mask = IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_SEND
| IPC_OBJECT_COPYIN_FLAGS_ALLOW_CONN_IMMOVABLE_RECEIVE;
copyin_mask = (copyin_flags & copyin_mask) | IPC_OBJECT_COPYIN_FLAGS_DEADOK;
/*
* We allow moving of immovable receive right of a service port when it is from launchd.
*/
task_t task = current_task_early();
#ifdef MACH_BSD
if (task && proc_isinitproc(get_bsdtask_info(task))) {
copyin_mask |= IPC_OBJECT_COPYIN_FLAGS_ALLOW_IMMOVABLE_RECEIVE;
}
#endif
/*
* Could first try a read lock when doing
* MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND,
* and MACH_MSG_TYPE_MAKE_SEND_ONCE.
*/
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS) {
return kr;
}
/* space is write-locked and active */
release_port = IP_NULL;
kr = ipc_right_copyin(space, name, entry,
msgt_name, copyin_mask,
objectp, &soright,
&release_port,
&assertcnt,
context,
guard_flags);
is_write_unlock(space);
if (moved_provisional_reply_port(msgt_name, soright)) {
send_prp_telemetry(-1);
}
#if IMPORTANCE_INHERITANCE
if (0 < assertcnt && ipc_importance_task_is_any_receiver_type(current_task()->task_imp_base)) {
ipc_importance_task_drop_internal_assertion(current_task()->task_imp_base, assertcnt);
}
#endif /* IMPORTANCE_INHERITANCE */
if (release_port != IP_NULL) {
ip_release(release_port);
}
if ((kr == KERN_SUCCESS) && (soright != IP_NULL)) {
ipc_notify_port_deleted(soright, name);
}
return kr;
}
/*
* Routine: ipc_object_copyin_from_kernel
* Purpose:
* Copyin a naked capability from the kernel.
*
* MACH_MSG_TYPE_MOVE_RECEIVE
* The receiver must be ipc_space_kernel
* or the receive right must already be in limbo.
* Consumes the naked receive right.
* MACH_MSG_TYPE_COPY_SEND
* A naked send right must be supplied.
* The port gains a reference, and a send right
* if the port is still active.
* MACH_MSG_TYPE_MAKE_SEND
* The receiver must be ipc_space_kernel.
* The port gains a reference and a send right.
* MACH_MSG_TYPE_MOVE_SEND
* Consumes a naked send right.
* MACH_MSG_TYPE_MAKE_SEND_ONCE
* The port gains a reference and a send-once right.
* Receiver also be the caller of device subsystem,
* so no assertion.
* MACH_MSG_TYPE_MOVE_SEND_ONCE
* Consumes a naked send-once right.
* Conditions:
* Nothing locked.
*/
void
ipc_object_copyin_from_kernel(
ipc_object_t object,
mach_msg_type_name_t msgt_name)
{
assert(IO_VALID(object));
switch (msgt_name) {
case MACH_MSG_TYPE_MOVE_RECEIVE: {
ipc_port_t port = ip_object_to_port(object);
ip_mq_lock(port);
require_ip_active(port);
if (ip_in_a_space(port)) {
assert(ip_in_space(port, ipc_space_kernel));
assert(port->ip_immovable_receive == 0);
/* relevant part of ipc_port_clear_receiver */
port->ip_mscount = 0;
/* port transtions to IN-LIMBO state */
port->ip_receiver_name = MACH_PORT_NULL;
port->ip_destination = IP_NULL;
}
ip_mq_unlock(port);
break;
}
case MACH_MSG_TYPE_COPY_SEND: {
ipc_port_t port = ip_object_to_port(object);
ip_mq_lock(port);
if (ip_active(port)) {
assert(port->ip_srights > 0);
}
ip_srights_inc(port);
ip_reference(port);
ip_mq_unlock(port);
break;
}
case MACH_MSG_TYPE_MAKE_SEND: {
ipc_port_t port = ip_object_to_port(object);
ip_mq_lock(port);
if (ip_active(port)) {
assert(ip_in_a_space(port));
assert((ip_in_space(port, ipc_space_kernel)) ||
(port->ip_receiver->is_node_id != HOST_LOCAL_NODE));
port->ip_mscount++;
}
ip_srights_inc(port);
ip_reference(port);
ip_mq_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND: {
/* move naked send right into the message */
assert(ip_object_to_port(object)->ip_srights);
break;
}
case MACH_MSG_TYPE_MAKE_SEND_ONCE: {
ipc_port_t port = ip_object_to_port(object);
ip_mq_lock(port);
if (ip_active(port)) {
assert(ip_in_a_space(port));
}
ipc_port_make_sonce_locked(port);
ip_mq_unlock(port);
break;
}
case MACH_MSG_TYPE_MOVE_SEND_ONCE: {
/* move naked send-once right into the message */
assert(ip_object_to_port(object)->ip_sorights);
break;
}
default:
panic("ipc_object_copyin_from_kernel: strange rights");
}
}
/*
* Routine: ipc_object_destroy
* Purpose:
* Destroys a naked capability.
* Consumes a ref for the object.
*
* A receive right should be in limbo or in transit.
* Conditions:
* Nothing locked.
*/
void
ipc_object_destroy(
ipc_object_t object,
mach_msg_type_name_t msgt_name)
{
ipc_port_t port = ip_object_to_port(object);
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
switch (msgt_name) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(port);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ip_mq_lock(port);
ipc_notify_send_once_and_unlock(port);
break;
case MACH_MSG_TYPE_PORT_RECEIVE:
ipc_port_release_receive(port);
break;
default:
panic("ipc_object_destroy: strange rights");
}
}
/*
* Routine: ipc_object_destroy_dest
* Purpose:
* Destroys a naked capability for the destination of
* of a message. Consumes a ref for the object.
*
* Conditions:
* Nothing locked.
*/
void
ipc_object_destroy_dest(
ipc_object_t object,
mach_msg_type_name_t msgt_name)
{
ipc_port_t port = ip_object_to_port(object);
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
switch (msgt_name) {
case MACH_MSG_TYPE_PORT_SEND:
ipc_port_release_send(port);
break;
case MACH_MSG_TYPE_PORT_SEND_ONCE:
ip_mq_lock(port);
ipc_notify_send_once_and_unlock(port);
break;
default:
panic("ipc_object_destroy_dest: strange rights");
}
}
/*
* Routine: ipc_object_insert_send_right
* Purpose:
* Insert a send right into an object already in the space.
* The specified name must already point to a valid object.
*
* Note: This really is a combined copyin()/copyout(),
* that avoids most of the overhead of being implemented that way.
*
* This is the fastpath for mach_port_insert_right.
*
* Conditions:
* Nothing locked.
*
* msgt_name must be MACH_MSG_TYPE_MAKE_SEND or
* MACH_MSG_TYPE_COPY_SEND.
*
* Returns:
* KERN_SUCCESS Copied out object, consumed ref.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_NAME Name doesn't exist in space.
* KERN_INVALID_CAPABILITY The object is dead.
* KERN_RIGHT_EXISTS Space has rights under another name.
*/
kern_return_t
ipc_object_insert_send_right(
ipc_space_t space,
mach_port_name_t name,
mach_msg_type_name_t msgt_name)
{
ipc_entry_bits_t bits;
ipc_object_t object;
ipc_entry_t entry;
ipc_port_t port;
kern_return_t kr;
assert(msgt_name == MACH_MSG_TYPE_MAKE_SEND ||
msgt_name == MACH_MSG_TYPE_COPY_SEND);
kr = ipc_right_lookup_write(space, name, &entry);
if (kr != KERN_SUCCESS) {
return kr;
}
/* space is write-locked and active */
bits = entry->ie_bits;
object = entry->ie_object;
if (!IO_VALID(object)) {
is_write_unlock(space);
return KERN_INVALID_CAPABILITY;
}
if ((bits & MACH_PORT_TYPE_PORT_RIGHTS) == 0) {
is_write_unlock(space);
return KERN_INVALID_RIGHT;
}
port = ip_object_to_port(object);
ip_mq_lock(port);
if (!ip_active(port)) {
kr = KERN_INVALID_CAPABILITY;
} else if (msgt_name == MACH_MSG_TYPE_MAKE_SEND) {
if (bits & MACH_PORT_TYPE_RECEIVE) {
port->ip_mscount++;
if ((bits & MACH_PORT_TYPE_SEND) == 0) {
ip_srights_inc(port);
bits |= MACH_PORT_TYPE_SEND;
}
/* leave urefs pegged to maximum if it overflowed */
if (IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX) {
bits += 1; /* increment urefs */
}
entry->ie_bits = bits;
ipc_entry_modified(space, name, entry);
kr = KERN_SUCCESS;
} else {
kr = KERN_INVALID_RIGHT;
}
} else { // MACH_MSG_TYPE_COPY_SEND
if (bits & MACH_PORT_TYPE_SEND) {
/* leave urefs pegged to maximum if it overflowed */
if (IE_BITS_UREFS(bits) < MACH_PORT_UREFS_MAX) {
entry->ie_bits = bits + 1; /* increment urefs */
}
ipc_entry_modified(space, name, entry);
kr = KERN_SUCCESS;
} else {
kr = KERN_INVALID_RIGHT;
}
}
ip_mq_unlock(port);
is_write_unlock(space);
return kr;
}
/*
* Routine: ipc_object_copyout
* Purpose:
* Copyout a capability, placing it into a space.
* Always consumes a ref for the object.
* Conditions:
* Nothing locked.
* Returns:
* KERN_SUCCESS Copied out object, consumed ref.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_CAPABILITY The object is dead.
* KERN_NO_SPACE No room in space for another right.
* KERN_UREFS_OVERFLOW Urefs limit exceeded
* and overflow wasn't specified.
*/
kern_return_t
ipc_object_copyout(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
ipc_object_copyout_flags_t flags,
mach_port_context_t *context,
mach_msg_guard_flags_t *guard_flags,
mach_port_name_t *namep)
{
struct knote *kn = current_thread()->ith_knote;
mach_port_name_t name;
ipc_port_t port = ip_object_to_port(object);
ipc_entry_t entry;
kern_return_t kr;
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
if (ITH_KNOTE_VALID(kn, msgt_name)) {
filt_machport_turnstile_prepare_lazily(kn, msgt_name, port);
}
is_write_lock(space);
for (;;) {
ipc_port_t port_subst = IP_NULL;
if (!is_active(space)) {
is_write_unlock(space);
kr = KERN_INVALID_TASK;
goto out;
}
kr = ipc_entries_hold(space, 1);
if (kr != KERN_SUCCESS) {
/* unlocks/locks space, so must start again */
kr = ipc_entry_grow_table(space, ITS_SIZE_NONE);
if (kr != KERN_SUCCESS) {
/* space is unlocked */
goto out;
}
continue;
}
ip_mq_lock_check_aligned(port);
if (!ip_active(port)) {
ip_mq_unlock(port);
is_write_unlock(space);
kr = KERN_INVALID_CAPABILITY;
goto out;
}
/* Don't actually copyout rights we aren't allowed to */
if (!ip_label_check(space, port, msgt_name, &flags, &port_subst)) {
ip_mq_unlock(port);
is_write_unlock(space);
assert(port_subst == IP_NULL);
kr = KERN_INVALID_CAPABILITY;
goto out;
}
/* is the kolabel requesting a substitution */
if (port_subst != IP_NULL) {
/*
* port is unlocked, its right consumed
* space is unlocked
*/
assert(msgt_name == MACH_MSG_TYPE_PORT_SEND);
port = port_subst;
if (!IP_VALID(port)) {
object = IO_DEAD;
kr = KERN_INVALID_CAPABILITY;
goto out;
}
object = ip_to_object(port);
is_write_lock(space);
continue;
}
break;
}
/* space is write-locked and active, object is locked and active */
if ((msgt_name != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
ipc_right_reverse(space, object, &name, &entry)) {
assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
} else {
ipc_entry_claim(space, object, &name, &entry);
}
kr = ipc_right_copyout(space, name, entry,
msgt_name, flags, context, guard_flags, object);
/* object is unlocked */
is_write_unlock(space);
out:
if (kr == KERN_SUCCESS) {
*namep = name;
} else if (IO_VALID(object)) {
ipc_object_destroy(object, msgt_name);
}
return kr;
}
/*
* Routine: ipc_object_copyout_name
* Purpose:
* Copyout a capability, placing it into a space.
* The specified name is used for the capability.
* If successful, consumes a ref for the object.
* Conditions:
* Nothing locked.
* Returns:
* KERN_SUCCESS Copied out object, consumed ref.
* KERN_INVALID_TASK The space is dead.
* KERN_INVALID_CAPABILITY The object is dead.
* KERN_UREFS_OVERFLOW Urefs limit exceeded
* and overflow wasn't specified.
* KERN_RIGHT_EXISTS Space has rights under another name.
* KERN_NAME_EXISTS Name is already used.
* KERN_INVALID_VALUE Supplied port name is invalid.
*/
kern_return_t
ipc_object_copyout_name(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
mach_port_name_t name)
{
ipc_port_t port = ip_object_to_port(object);
mach_port_name_t oname;
ipc_entry_t oentry;
ipc_entry_t entry;
kern_return_t kr;
#if IMPORTANCE_INHERITANCE
int assertcnt = 0;
ipc_importance_task_t task_imp = IIT_NULL;
#endif /* IMPORTANCE_INHERITANCE */
assert(IO_VALID(object));
assert(io_otype(object) == IOT_PORT);
kr = ipc_entry_alloc_name(space, name, &entry);
if (kr != KERN_SUCCESS) {
return kr;
}
/* space is write-locked and active */
ip_mq_lock_check_aligned(port);
/*
* Don't actually copyout rights we aren't allowed to
*
* In particular, kolabel-ed objects do not allow callers
* to pick the name they end up with.
*/
if (!ip_active(port) || ip_is_kolabeled(port)) {
ip_mq_unlock(port);
if (!ipc_right_inuse(entry)) {
ipc_entry_dealloc(space, IO_NULL, name, entry);
}
is_write_unlock(space);
return KERN_INVALID_CAPABILITY;
}
/* space is write-locked and active, object is locked and active */
if ((msgt_name != MACH_MSG_TYPE_PORT_SEND_ONCE) &&
ipc_right_reverse(space, object, &oname, &oentry)) {
if (name != oname) {
ip_mq_unlock(port);
if (!ipc_right_inuse(entry)) {
ipc_entry_dealloc(space, IO_NULL, name, entry);
}
is_write_unlock(space);
return KERN_RIGHT_EXISTS;
}
assert(entry == oentry);
assert(entry->ie_bits & MACH_PORT_TYPE_SEND_RECEIVE);
} else if (ipc_right_inuse(entry)) {
ip_mq_unlock(port);
is_write_unlock(space);
return KERN_NAME_EXISTS;
} else {
assert(entry->ie_object == IO_NULL);
entry->ie_object = object;
}
#if IMPORTANCE_INHERITANCE
/*
* We are slamming a receive right into the space, without
* first having been enqueued on a port destined there. So,
* we have to arrange to boost the task appropriately if this
* port has assertions (and the task wants them).
*/
if (msgt_name == MACH_MSG_TYPE_PORT_RECEIVE) {
if (space->is_task != TASK_NULL) {
task_imp = space->is_task->task_imp_base;
if (ipc_importance_task_is_any_receiver_type(task_imp)) {
assertcnt = port->ip_impcount;
ipc_importance_task_reference(task_imp);
} else {
task_imp = IIT_NULL;
}
}
/* take port out of limbo */
port->ip_tempowner = 0;
}
#endif /* IMPORTANCE_INHERITANCE */
kr = ipc_right_copyout(space, name, entry,
msgt_name, IPC_OBJECT_COPYOUT_FLAGS_NONE, NULL, NULL, object);
/* object is unlocked */
is_write_unlock(space);
#if IMPORTANCE_INHERITANCE
/*
* Add the assertions to the task that we captured before
*/
if (task_imp != IIT_NULL) {
ipc_importance_task_hold_internal_assertion(task_imp, assertcnt);
ipc_importance_task_release(task_imp);
}
#endif /* IMPORTANCE_INHERITANCE */
return kr;
}
/*
* Routine: ipc_object_copyout_dest
* Purpose:
* Translates/consumes the destination right of a message.
* This is unlike normal copyout because the right is consumed
* in a funny way instead of being given to the receiving space.
* The receiver gets his name for the port, if he has receive
* rights, otherwise MACH_PORT_NULL.
* Conditions:
* The object is locked and active. Nothing else locked.
* The object is unlocked and loses a reference.
*/
void
ipc_object_copyout_dest(
ipc_space_t space,
ipc_object_t object,
mach_msg_type_name_t msgt_name,
mach_port_name_t *namep)
{
mach_port_name_t name;
assert(IO_VALID(object));
assert(io_active(object));
/*
* If the space is the receiver/owner of the object,
* then we quietly consume the right and return
* the space's name for the object. Otherwise
* we destroy the right and return MACH_PORT_NULL.
*/
switch (msgt_name) {
case MACH_MSG_TYPE_PORT_SEND: {
ipc_port_t port = ip_object_to_port(object);
ipc_notify_nsenders_t nsrequest = { };
if (ip_in_space(port, space)) {
name = ip_get_receiver_name(port);
} else {
name = MACH_PORT_NULL;
}
ip_srights_dec(port);
if (port->ip_srights == 0) {
nsrequest = ipc_notify_no_senders_prepare(port);
}
ipc_port_clear_sync_rcv_thread_boost_locked(port);
/* port unlocked */
ipc_notify_no_senders_emit(nsrequest);
ip_release(port);
break;
}
case MACH_MSG_TYPE_PORT_SEND_ONCE: {
ipc_port_t port = ip_object_to_port(object);
assert(port->ip_sorights > 0);
if (ip_in_space(port, space)) {
/* quietly consume the send-once right */
ip_sorights_dec(port);
name = ip_get_receiver_name(port);
ipc_port_clear_sync_rcv_thread_boost_locked(port);
/* port unlocked */
ip_release(port);
} else {
/*
* A very bizarre case. The message
* was received, but before this copyout
* happened the space lost receive rights.
* We can't quietly consume the soright
* out from underneath some other task,
* so generate a send-once notification.
*/
ipc_notify_send_once_and_unlock(port);
name = MACH_PORT_NULL;
}
break;
}
default:
panic("ipc_object_copyout_dest: strange rights");
name = MACH_PORT_DEAD;
}
*namep = name;
}
static_assert(offsetof(struct ipc_object_waitq, iowq_waitq) ==
offsetof(struct ipc_port, ip_waitq));
static_assert(offsetof(struct ipc_object_waitq, iowq_waitq) ==
offsetof(struct ipc_pset, ips_wqset));
/*
* Routine: ipc_object_lock
* Purpose:
* Validate, then acquire a lock on an ipc object
*/
void
ipc_object_lock(ipc_object_t io, ipc_object_type_t type)
{
ipc_object_validate(io, type);
waitq_lock(io_waitq(io));
}
void
ipc_object_lock_check_aligned(ipc_object_t io, ipc_object_type_t type)
{
ipc_object_validate_aligned(io, type);
waitq_lock(io_waitq(io));
}
__abortlike
static void
ipc_object_validate_preflight_panic(ipc_object_t io)
{
panic("ipc object %p is neither a port or a port-set", io);
}
/*
* Routine: ipc_object_lock_allow_invalid
* Purpose:
* Speculatively try to lock an object in an undefined state.
*
* This relies on the fact that IPC object memory is allocated
* from sequestered zones, so at a given address, one can find:
* 1. a valid object,
* 2. a freed or invalid (uninitialized) object,
* 3. unmapped memory.
*
* (2) is possible because the zone is made with ZC_ZFREE_CLEARMEM which
* ensures freed elements are always zeroed.
*
* (3) is a direct courtesy of waitq_lock_allow_invalid().
*
* In order to disambiguate (1) from (2), we use the "waitq valid"
* bit which is part of the lock. When that bit is absent,
* waitq_lock() will function as expected, but
* waitq_lock_allow_invalid() will not.
*
* Objects are then initialized and destroyed carefully so that
* this "valid bit" is only set when the object invariants are
* respected.
*
* Returns:
* true: the lock was acquired
* false: the object was freed or not initialized.
*/
bool
ipc_object_lock_allow_invalid(ipc_object_t orig_io)
{
struct waitq *orig_wq = io_waitq(orig_io);
struct waitq *wq = pgz_decode_allow_invalid(orig_wq, ZONE_ID_ANY);
switch (zone_id_for_element(wq, sizeof(*wq))) {
case ZONE_ID_IPC_PORT:
case ZONE_ID_IPC_PORT_SET:
break;
default:
#if CONFIG_PROB_GZALLOC
if (orig_wq != wq) {
/*
* The element was PGZ protected, and the translation
* returned another type than port or port-set, or
* ZONE_ID_INVALID (wq is NULL).
*
* We have to allow this skew, and assumed the slot
* has held a now freed port/port-set.
*/
return false;
}
#endif /* CONFIG_PROB_GZALLOC */
ipc_object_validate_preflight_panic(orig_io);
}
if (__probable(waitq_lock_allow_invalid(wq))) {
ipc_object_t io = io_from_waitq(wq);
ipc_object_validate(io, io_otype(io));
#if CONFIG_PROB_GZALLOC
if (__improbable(wq != orig_wq &&
wq != pgz_decode_allow_invalid(orig_wq, ZONE_ID_ANY))) {
/*
* This object is no longer held in the slot,
* whatever this object is, it's not the droid
* we're looking for. Pretend we failed the lock.
*/
waitq_unlock(wq);
return false;
}
#endif /* CONFIG_PROB_GZALLOC */
return true;
}
return false;
}
/*
* Routine: ipc_object_lock_try
* Purpose:
* Validate, then try to acquire a lock on an object,
* fail if there is an existing busy lock
*/
bool
ipc_object_lock_try(ipc_object_t io, ipc_object_type_t type)
{
ipc_object_validate(io, type);
return waitq_lock_try(io_waitq(io));
}
/*
* Routine: ipc_object_unlock
* Purpose:
* Unlocks the given object.
*/
void
ipc_object_unlock(ipc_object_t io)
{
waitq_unlock(io_waitq(io));
}