This is xnu-11215.1.10. See this file in:
/*
* Copyright (c) 2015-2024 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* if_fake.c
* - fake network interface used for testing
* - "feth" (e.g. "feth0", "feth1") is a virtual ethernet interface that allows
* two instances to have their output/input paths "crossed-over" so that
* output on one is input on the other
*/
/*
* Modification History:
*
* September 9, 2015 Dieter Siegmund (dieter@apple.com)
* - created
*/
#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/queue.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/kern_event.h>
#include <sys/mcache.h>
#include <sys/syslog.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_vlan_var.h>
#include <net/if_fake_var.h>
#include <net/if_arp.h>
#include <net/if_dl.h>
#include <net/if_ether.h>
#include <net/if_types.h>
#include <libkern/OSAtomic.h>
#include <net/dlil.h>
#include <net/kpi_interface.h>
#include <net/kpi_protocol.h>
#include <kern/locks.h>
#include <kern/zalloc.h>
#include <mach/mach_time.h>
#include <os/log.h>
#ifdef INET
#include <netinet/in.h>
#include <netinet/if_ether.h>
#endif
#include <net/if_media.h>
#include <net/ether_if_module.h>
#if SKYWALK
#include <skywalk/os_skywalk_private.h>
#include <skywalk/nexus/netif/nx_netif.h>
#include <skywalk/channel/channel_var.h>
#endif /* SKYWALK */
/*
* if_fake_debug, FE_DBGF_*
* - 'if_fake_debug' is a bitmask of FE_DBGF_* flags that can be set
* to enable additional logs for the corresponding fake function
* - "sysctl net.link.fake.debug" controls the value of
* 'if_fake_debug'
*/
static uint32_t if_fake_debug = 0;
#define FE_DBGF_LIFECYCLE 0x0001
#define FE_DBGF_INPUT 0x0002
#define FE_DBGF_OUTPUT 0x0004
#define FE_DBGF_CONTROL 0x0008
#define FE_DBGF_MISC 0x0010
/*
* if_fake_log_level
* - 'if_fake_log_level' ensures that by default important logs are
* logged regardless of if_fake_debug by comparing the log level
* in FAKE_LOG to if_fake_log_level
* - use "sysctl net.link.fake.log_level" controls the value of
* 'if_fake_log_level'
* - the default value of 'if_fake_log_level' is LOG_NOTICE; important
* logs must use LOG_NOTICE to ensure they appear by default
*/
#define FAKE_DBGF_ENABLED(__flag) ((if_fake_debug & __flag) != 0)
/*
* FAKE_LOG
* - macro to generate the specified log conditionally based on
* the specified log level and debug flags
*/
#define FAKE_LOG(__level, __dbgf, __string, ...) \
do { \
if (__level <= if_fake_log_level || \
FAKE_DBGF_ENABLED(__dbgf)) { \
os_log(OS_LOG_DEFAULT, "%s: " __string, \
__func__, ## __VA_ARGS__); \
} \
} while (0)
static boolean_t
is_power_of_two(unsigned int val)
{
return (val & (val - 1)) == 0;
}
#define FAKE_ETHER_NAME "feth"
SYSCTL_DECL(_net_link);
SYSCTL_NODE(_net_link, OID_AUTO, fake, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
"Fake interface");
static int if_fake_txstart = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, txstart, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_txstart, 0, "Fake interface TXSTART mode");
static int if_fake_hwcsum = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, hwcsum, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_hwcsum, 0, "Fake interface simulate hardware checksum");
static int if_fake_vlan_tagging = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, vlan_tagging, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_vlan_tagging, 0, "Fake interface VLAN tagging");
static int if_fake_nxattach = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, nxattach, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_nxattach, 0, "Fake interface auto-attach nexus");
static int if_fake_bsd_mode = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, bsd_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_bsd_mode, 0, "Fake interface attach as BSD interface");
static int if_fake_log_level = LOG_NOTICE;
SYSCTL_INT(_net_link_fake, OID_AUTO, log_level, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_log_level, 0, "Fake interface log level");
SYSCTL_INT(_net_link_fake, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_debug, 0, "Fake interface debug flags");
static int if_fake_wmm_mode = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, wmm_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_wmm_mode, 0, "Fake interface in 802.11 WMM mode");
static int if_fake_multibuflet = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, multibuflet, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_multibuflet, 0, "Fake interface using multi-buflet packets");
static int if_fake_low_latency = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, low_latency, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_low_latency, 0, "Fake interface with a low latency qset");
static int if_fake_switch_combined_mode = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, switch_combined_mode,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_fake_switch_combined_mode, 0,
"Switch a qset between combined and separate mode during dequeues");
static int if_fake_switch_mode_frequency = 10;
SYSCTL_INT(_net_link_fake, OID_AUTO, switch_mode_frequency,
CTLFLAG_RW | CTLFLAG_LOCKED, &if_fake_switch_mode_frequency, 0,
"The number of dequeues before we switch between the combined and separated mode");
static int if_fake_tso_support = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, tso_support, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_tso_support, 0, "Fake interface with support for TSO offload");
#define DEFAULT_EXPIRATION_THRESHOLD 500 /* usec */
static int if_fake_expiration_threshold_us = DEFAULT_EXPIRATION_THRESHOLD;
SYSCTL_INT(_net_link_fake, OID_AUTO, expiration_threshold, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_expiration_threshold_us, DEFAULT_EXPIRATION_THRESHOLD,
"Expiration threshold (usec) for expiration testing");
static int if_fake_lro = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_lro, 0, "Fake interface report LRO capability");
static int if_fake_separate_frame_header = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, separate_frame_header,
CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_separate_frame_header, 0, "Put frame header in separate mbuf");
typedef enum {
IFF_PP_MODE_GLOBAL = 0, /* share a global pool */
IFF_PP_MODE_PRIVATE = 1, /* creates its own rx/tx pool */
IFF_PP_MODE_PRIVATE_SPLIT = 2, /* creates its own split rx & tx pool */
} iff_pktpool_mode_t;
static iff_pktpool_mode_t if_fake_pktpool_mode = IFF_PP_MODE_GLOBAL;
SYSCTL_INT(_net_link_fake, OID_AUTO, pktpool_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_pktpool_mode, IFF_PP_MODE_GLOBAL,
"Fake interface packet pool mode (0 global, 1 private, 2 private split");
#define FETH_LINK_LAYER_AGGRETATION_FACTOR_MAX 512
#define FETH_LINK_LAYER_AGGRETATION_FACTOR_DEF 96
static int if_fake_link_layer_aggregation_factor =
FETH_LINK_LAYER_AGGRETATION_FACTOR_DEF;
static int
feth_link_layer_aggregation_factor_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_link_layer_aggregation_factor,
sizeof(if_fake_link_layer_aggregation_factor), &new_value,
&changed);
if (error == 0 && changed != 0) {
if (new_value <= 0 ||
new_value > FETH_LINK_LAYER_AGGRETATION_FACTOR_MAX) {
return EINVAL;
}
if_fake_link_layer_aggregation_factor = new_value;
}
return error;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, link_layer_aggregation_factor,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_link_layer_aggregation_factor_sysctl, "IU",
"Fake interface link layer aggregation factor");
#define FETH_TX_HEADROOM_MAX 32
static unsigned int if_fake_tx_headroom = FETH_TX_HEADROOM_MAX;
static int
feth_tx_headroom_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_tx_headroom,
sizeof(if_fake_tx_headroom), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > FETH_TX_HEADROOM_MAX ||
(new_value % 8) != 0) {
return EINVAL;
}
if_fake_tx_headroom = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_headroom,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_tx_headroom_sysctl, "IU", "Fake ethernet Tx headroom");
static int if_fake_fcs = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, fcs, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_fcs, 0, "Fake interface using frame check sequence");
#define FETH_TRAILER_LENGTH_MAX 28
char feth_trailer[FETH_TRAILER_LENGTH_MAX + 1] = "trailertrailertrailertrailer";
static unsigned int if_fake_trailer_length = 0;
static int
feth_trailer_length_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_trailer_length,
sizeof(if_fake_trailer_length), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > FETH_TRAILER_LENGTH_MAX) {
return EINVAL;
}
if_fake_trailer_length = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, trailer_length,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_trailer_length_sysctl, "IU", "Fake interface frame trailer length");
/* sysctl net.link.fake.max_mtu */
#define FETH_MAX_MTU_DEFAULT 2048
#define FETH_MAX_MTU_MAX ((16 * 1024) - ETHER_HDR_LEN)
static unsigned int if_fake_max_mtu = FETH_MAX_MTU_DEFAULT;
/* sysctl net.link.fake.buflet_size */
#define FETH_BUFLET_SIZE_MIN 512
#define FETH_BUFLET_SIZE_MAX (32 * 1024)
#define FETH_TSO_BUFLET_SIZE (16 * 1024)
static unsigned int if_fake_buflet_size = FETH_BUFLET_SIZE_MIN;
static unsigned int if_fake_tso_buffer_size = FETH_TSO_BUFLET_SIZE;
static int
feth_tso_buffer_size_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_tso_buffer_size,
sizeof(if_fake_tso_buffer_size), &new_value, &changed);
if (error == 0 && changed != 0) {
/* must be a power of 2 between min and max */
if (new_value > FETH_BUFLET_SIZE_MAX ||
new_value < FETH_BUFLET_SIZE_MIN ||
!is_power_of_two(new_value)) {
return EINVAL;
}
if_fake_tso_buffer_size = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tso_buf_size,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_tso_buffer_size_sysctl, "IU", "Fake interface TSO buffer size");
static int
feth_max_mtu_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_max_mtu,
sizeof(if_fake_max_mtu), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > FETH_MAX_MTU_MAX ||
new_value < ETHERMTU ||
new_value <= if_fake_buflet_size) {
return EINVAL;
}
if_fake_max_mtu = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, max_mtu,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_max_mtu_sysctl, "IU", "Fake interface maximum MTU");
static int
feth_buflet_size_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_buflet_size,
sizeof(if_fake_buflet_size), &new_value, &changed);
if (error == 0 && changed != 0) {
/* must be a power of 2 between min and max */
if (new_value > FETH_BUFLET_SIZE_MAX ||
new_value < FETH_BUFLET_SIZE_MIN ||
!is_power_of_two(new_value) ||
new_value >= if_fake_max_mtu) {
return EINVAL;
}
if_fake_buflet_size = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, buflet_size,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_buflet_size_sysctl, "IU", "Fake interface buflet size");
static unsigned int if_fake_user_access = 0;
static int
feth_user_access_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_user_access,
sizeof(if_fake_user_access), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value != 0) {
if (new_value != 1) {
return EINVAL;
}
}
if_fake_user_access = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, user_access,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
0, 0, feth_user_access_sysctl, "IU", "Fake interface user access");
/* sysctl net.link.fake.if_adv_intvl (unit: millisecond) */
#define FETH_IF_ADV_INTVL_MIN 10
#define FETH_IF_ADV_INTVL_MAX INT_MAX
static int if_fake_if_adv_interval = 0; /* no interface advisory */
static int
feth_if_adv_interval_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_if_adv_interval,
sizeof(if_fake_if_adv_interval), &new_value, &changed);
if (error == 0 && changed != 0) {
if ((new_value != 0) && (new_value > FETH_IF_ADV_INTVL_MAX ||
new_value < FETH_IF_ADV_INTVL_MIN)) {
return EINVAL;
}
if_fake_if_adv_interval = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, if_adv_intvl,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_if_adv_interval_sysctl, "IU",
"Fake interface will generate interface advisories reports at the specified interval in ms");
/* sysctl net.link.fake.tx_drops */
/*
* Fake ethernet will drop packet on the transmit path at the specified
* rate, i.e drop one in every if_fake_tx_drops number of packets.
*/
#define FETH_TX_DROPS_MIN 0
#define FETH_TX_DROPS_MAX INT_MAX
static int if_fake_tx_drops = 0; /* no packets are dropped */
static int
feth_fake_tx_drops_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_tx_drops,
sizeof(if_fake_tx_drops), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > FETH_TX_DROPS_MAX ||
new_value < FETH_TX_DROPS_MIN) {
return EINVAL;
}
if_fake_tx_drops = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_drops,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_fake_tx_drops_sysctl, "IU",
"Fake interface will intermittently drop packets on Tx path");
/* sysctl.net.link.fake.tx_exp_policy */
typedef enum {
IFF_TX_EXP_POLICY_DISABLED = 0, /* Expiry notification disabled */
IFF_TX_EXP_POLICY_DROP_AND_NOTIFY = 1, /* Expiry notification enabled; drop + notify mode */
IFF_TX_EXP_POLICY_NOTIFY_ONLY = 2, /* Expiry notification enabled; notify only mode */
IFF_TX_EXP_POLICY_METADATA = 3, /* Expiry notification enabled; use packet metadata */
} iff_tx_exp_policy_t;
static iff_tx_exp_policy_t if_fake_tx_exp_policy = IFF_TX_EXP_POLICY_DISABLED;
static int
feth_fake_tx_exp_policy_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_tx_exp_policy,
sizeof(if_fake_tx_exp_policy), &new_value, &changed);
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"if_fake_tx_exp_policy: %u -> %u (%d)",
if_fake_tx_exp_policy, new_value, changed);
if (error == 0 && changed != 0) {
if (new_value > IFF_TX_EXP_POLICY_METADATA ||
new_value < IFF_TX_EXP_POLICY_DISABLED) {
return EINVAL;
}
if_fake_tx_exp_policy = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_exp_policy,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_fake_tx_exp_policy_sysctl, "IU",
"Fake interface handling policy for expired TX attempts "
"(0 disabled, 1 drop and notify, 2 notify only, 3 packet metadata)");
/* sysctl net.link.fake.tx_completion_mode */
typedef enum {
IFF_TX_COMPL_MODE_SYNC = 0,
IFF_TX_COMPL_MODE_ASYNC = 1,
} iff_tx_completion_mode_t;
static iff_tx_completion_mode_t if_tx_completion_mode = IFF_TX_COMPL_MODE_SYNC;
static int
feth_fake_tx_completion_mode_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_tx_completion_mode,
sizeof(if_tx_completion_mode), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > IFF_TX_COMPL_MODE_ASYNC ||
new_value < IFF_TX_COMPL_MODE_SYNC) {
return EINVAL;
}
if_tx_completion_mode = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_completion_mode,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_fake_tx_completion_mode_sysctl, "IU",
"Fake interface tx completion mode (0 synchronous, 1 asynchronous)");
/* sysctl net.link.fake.llink_cnt */
/* The maximum number of logical links (including default link) */
#define FETH_MAX_LLINKS 16
/*
* The default number of logical links (including default link).
* Zero means logical link mode is disabled.
*/
#define FETH_DEF_LLINKS 0
static uint32_t if_fake_llink_cnt = FETH_DEF_LLINKS;
static int
feth_fake_llink_cnt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_llink_cnt,
sizeof(if_fake_llink_cnt), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value > FETH_MAX_LLINKS) {
return EINVAL;
}
if_fake_llink_cnt = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, llink_cnt,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_fake_llink_cnt_sysctl, "IU",
"Fake interface logical link count");
/* sysctl net.link.fake.qset_cnt */
/* The maximum number of qsets for each logical link */
#define FETH_MAX_QSETS 16
/* The default number of qsets for each logical link */
#define FETH_DEF_QSETS 4
static uint32_t if_fake_qset_cnt = FETH_DEF_QSETS;
static int
feth_fake_qset_cnt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
unsigned int new_value;
int changed;
int error;
error = sysctl_io_number(req, if_fake_qset_cnt,
sizeof(if_fake_qset_cnt), &new_value, &changed);
if (error == 0 && changed != 0) {
if (new_value == 0 ||
new_value > FETH_MAX_QSETS) {
return EINVAL;
}
if_fake_qset_cnt = new_value;
}
return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, qset_cnt,
CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
feth_fake_qset_cnt_sysctl, "IU",
"Fake interface queue set count");
static void
_mbuf_adjust_pkthdr_and_data(mbuf_t m, int len)
{
mbuf_setdata(m, (char *)mbuf_data(m) + len, mbuf_len(m) - len);
mbuf_pkthdr_adjustlen(m, -len);
}
static inline void *
get_bpf_header(mbuf_t m, struct ether_header * eh_p,
struct ether_vlan_header * evl_p, size_t * header_len)
{
void * header;
/* no VLAN tag, just use the ethernet header */
if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
header = eh_p;
*header_len = sizeof(*eh_p);
goto done;
}
/* has VLAN tag, populate the ether VLAN header */
bcopy(eh_p, evl_p,
offsetof(struct ether_header, ether_type)); /* dst+src ether */
evl_p->evl_encap_proto = htons(ETHERTYPE_VLAN); /* VLAN encap */
evl_p->evl_tag = htons(m->m_pkthdr.vlan_tag); /* tag */
evl_p->evl_proto = eh_p->ether_type; /* proto */
*header_len = sizeof(*evl_p);
header = evl_p;
done:
return header;
}
typedef void (*_tap_func)(ifnet_t interface, u_int32_t dlt, mbuf_t packet,
void *__sized_by(header_len) header, size_t header_len);
static void
fake_bpf_tap_common(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p,
_tap_func func)
{
struct ether_vlan_header evl;
void * header;
size_t header_len;
header = get_bpf_header(m, eh_p, &evl, &header_len);
(*func)(ifp, DLT_EN10MB, m, header, header_len);
}
static inline void
fake_bpf_tap_in(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p)
{
fake_bpf_tap_common(ifp, m, eh_p, bpf_tap_in);
}
static inline void
fake_bpf_tap_out(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p)
{
fake_bpf_tap_common(ifp, m, eh_p, bpf_tap_out);
}
/**
** virtual ethernet structures, types
**/
#define IFF_NUM_TX_RINGS_WMM_MODE 4
#define IFF_NUM_RX_RINGS_WMM_MODE 1
#define IFF_MAX_TX_RINGS IFF_NUM_TX_RINGS_WMM_MODE
#define IFF_MAX_RX_RINGS IFF_NUM_RX_RINGS_WMM_MODE
#define IFF_NUM_TX_QUEUES_WMM_MODE 4
#define IFF_NUM_RX_QUEUES_WMM_MODE 1
#define IFF_MAX_TX_QUEUES IFF_NUM_TX_QUEUES_WMM_MODE
#define IFF_MAX_RX_QUEUES IFF_NUM_RX_QUEUES_WMM_MODE
#define IFF_MAX_BATCH_SIZE 32
typedef uint16_t iff_flags_t;
#define IFF_FLAGS_HWCSUM 0x0001
#define IFF_FLAGS_BSD_MODE 0x0002
#define IFF_FLAGS_DETACHING 0x0004
#define IFF_FLAGS_WMM_MODE 0x0008
#define IFF_FLAGS_MULTIBUFLETS 0x0010
#define IFF_FLAGS_TSO_SUPPORT 0x0020
#define IFF_FLAGS_LRO 0x0040
#define IFF_FLAGS_VLAN_MTU 0x0080
#define IFF_FLAGS_VLAN_TAGGING 0x0100
#define IFF_FLAGS_SEPARATE_FRAME_HEADER 0x0200
#if SKYWALK
typedef struct {
uuid_t fnx_provider;
uuid_t fnx_instance;
} fake_nx, *fake_nx_t;
typedef struct {
kern_netif_queue_t fq_queue;
} fake_queue;
typedef struct {
kern_netif_qset_t fqs_qset; /* provided by xnu */
fake_queue fqs_rx_queue[IFF_MAX_RX_QUEUES];
fake_queue fqs_tx_queue[IFF_MAX_TX_QUEUES];
uint32_t fqs_rx_queue_cnt;
uint32_t fqs_tx_queue_cnt;
uint32_t fqs_llink_idx;
uint32_t fqs_idx;
uint32_t fqs_dequeue_cnt;
uint64_t fqs_id;
boolean_t fqs_combined_mode;
} fake_qset;
typedef struct {
uint64_t fl_id;
uint32_t fl_idx;
uint32_t fl_qset_cnt;
fake_qset fl_qset[FETH_MAX_QSETS];
} fake_llink;
static kern_pbufpool_t S_pp;
#define IFF_TT_OUTPUT 0x01 /* generate trace_tag on output */
#define IFF_TT_INPUT 0x02 /* generate trace_tag on input */
static int if_fake_trace_tag_flags = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, trace_tag, CTLFLAG_RW | CTLFLAG_LOCKED,
&if_fake_trace_tag_flags, 0, "Fake interface generate trace_tag");
static packet_trace_tag_t if_fake_trace_tag_current = 1;
#endif /* SKYWALK */
struct if_fake {
char iff_name[IFNAMSIZ]; /* our unique id */
ifnet_t iff_ifp;
iff_flags_t iff_flags;
uint32_t iff_retain_count;
ifnet_t iff_peer; /* the other end */
int iff_media_current;
int iff_media_active;
uint32_t iff_media_count;
int iff_media_list[IF_FAKE_MEDIA_LIST_MAX];
boolean_t iff_start_busy;
unsigned int iff_max_mtu;
uint32_t iff_fcs;
uint32_t iff_trailer_length;
#if SKYWALK
fake_nx iff_nx;
struct netif_stats *iff_nifs;
uint32_t iff_nifs_ref;
uint32_t iff_llink_cnt;
kern_channel_ring_t iff_rx_ring[IFF_MAX_RX_RINGS];
kern_channel_ring_t iff_tx_ring[IFF_MAX_TX_RINGS];
fake_llink *iff_llink __counted_by(FETH_MAX_LLINKS);
thread_call_t iff_doorbell_tcall;
thread_call_t iff_if_adv_tcall;
boolean_t iff_doorbell_tcall_active;
boolean_t iff_waiting_for_tcall;
boolean_t iff_channel_connected;
iff_pktpool_mode_t iff_pp_mode;
kern_pbufpool_t iff_rx_pp;
kern_pbufpool_t iff_tx_pp;
uint32_t iff_tx_headroom;
unsigned int iff_adv_interval;
uint32_t iff_tx_drop_rate;
uint32_t iff_tx_pkts_count;
iff_tx_completion_mode_t iff_tx_completion_mode;
bool iff_intf_adv_enabled;
void *iff_intf_adv_kern_ctx;
kern_nexus_capab_interface_advisory_notify_fn_t iff_intf_adv_notify;
iff_tx_exp_policy_t iff_tx_exp_policy;
#endif /* SKYWALK */
};
typedef struct if_fake * if_fake_ref;
static if_fake_ref
ifnet_get_if_fake(ifnet_t ifp);
static inline boolean_t
feth_in_bsd_mode(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_BSD_MODE) != 0;
}
static inline void
feth_set_detaching(if_fake_ref fakeif)
{
fakeif->iff_flags |= IFF_FLAGS_DETACHING;
}
static inline boolean_t
feth_is_detaching(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_DETACHING) != 0;
}
static int
feth_enable_dequeue_stall(ifnet_t ifp, uint32_t enable)
{
int error;
if (enable != 0) {
error = ifnet_disable_output(ifp);
} else {
error = ifnet_enable_output(ifp);
}
return error;
}
#if SKYWALK
static inline boolean_t
feth_in_wmm_mode(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_WMM_MODE) != 0;
}
static inline boolean_t
feth_using_multibuflets(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_MULTIBUFLETS) != 0;
}
static void feth_detach_netif_nexus(if_fake_ref fakeif);
static inline boolean_t
feth_has_intf_advisory_configured(if_fake_ref fakeif)
{
return fakeif->iff_adv_interval > 0;
}
#endif /* SKYWALK */
static inline bool
feth_supports_tso(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_TSO_SUPPORT) != 0;
}
static inline void
feth_set_supports_tso(if_fake_ref fakeif)
{
fakeif->iff_flags |= IFF_FLAGS_TSO_SUPPORT;
}
static inline bool
feth_supports_vlan_mtu(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_VLAN_MTU) != 0;
}
static inline void
feth_set_supports_vlan_mtu(if_fake_ref fakeif)
{
fakeif->iff_flags |= IFF_FLAGS_VLAN_MTU;
}
static inline bool
feth_supports_vlan_tagging(if_fake_ref fakeif)
{
return (fakeif->iff_flags & IFF_FLAGS_VLAN_TAGGING) != 0;
}
static inline void
feth_set_supports_vlan_tagging(if_fake_ref fakeif)
{
fakeif->iff_flags |= IFF_FLAGS_VLAN_TAGGING;
}
#define FETH_MAXUNIT IF_MAXUNIT
#define FETH_ZONE_MAX_ELEM MIN(IFNETS_MAX, FETH_MAXUNIT)
static int feth_clone_create(struct if_clone *, u_int32_t, void *);
static int feth_clone_destroy(ifnet_t);
static int feth_output(ifnet_t ifp, struct mbuf *m);
static void feth_start(ifnet_t ifp);
static int feth_ioctl(ifnet_t ifp, u_long cmd, void * addr);
static int feth_config(ifnet_t ifp, ifnet_t peer);
static void feth_if_free(ifnet_t ifp);
static void feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp);
static void feth_free(if_fake_ref fakeif);
static struct if_clone
feth_cloner = IF_CLONE_INITIALIZER(FAKE_ETHER_NAME,
feth_clone_create,
feth_clone_destroy,
0,
FETH_MAXUNIT);
static void interface_link_event(ifnet_t ifp, u_int32_t event_code);
/* some media words to pretend to be ethernet */
#define FAKE_DEFAULT_MEDIA IFM_MAKEWORD(IFM_ETHER, IFM_10G_T, IFM_FDX, 0)
static int default_media_words[] = {
IFM_MAKEWORD(IFM_ETHER, 0, 0, 0),
FAKE_DEFAULT_MEDIA,
IFM_MAKEWORD(IFM_ETHER, IFM_2500_T, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_5000_T, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_10G_KX4, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_20G_KR2, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_2500_SX, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_25G_KR, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_40G_SR4, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_50G_CR2, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_56G_R4, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_100G_CR4, IFM_FDX, 0),
IFM_MAKEWORD(IFM_ETHER, IFM_400G_AUI8, IFM_FDX, 0),
};
#define default_media_words_count (sizeof(default_media_words) \
/ sizeof (default_media_words[0]))
/**
** veth locks
**/
static LCK_GRP_DECLARE(feth_lck_grp, "fake");
static LCK_MTX_DECLARE(feth_lck_mtx, &feth_lck_grp);
static inline void
feth_lock(void)
{
lck_mtx_lock(&feth_lck_mtx);
}
static inline void
feth_unlock(void)
{
lck_mtx_unlock(&feth_lck_mtx);
}
static inline int
get_max_mtu(int bsd_mode, unsigned int max_mtu)
{
unsigned int mtu;
if (bsd_mode != 0) {
mtu = (njcl > 0) ? (M16KCLBYTES - ETHER_HDR_LEN)
: MBIGCLBYTES - ETHER_HDR_LEN;
if (mtu > max_mtu) {
mtu = max_mtu;
}
} else {
mtu = max_mtu;
}
return mtu;
}
static inline unsigned int
feth_max_mtu(ifnet_t ifp)
{
if_fake_ref fakeif;
unsigned int max_mtu = ETHERMTU;
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif != NULL) {
max_mtu = fakeif->iff_max_mtu;
}
feth_unlock();
return max_mtu;
}
static void
feth_free(if_fake_ref fakeif)
{
VERIFY(fakeif->iff_retain_count == 0);
#if SKYWALK
if (!feth_in_bsd_mode(fakeif)) {
if (fakeif->iff_pp_mode == IFF_PP_MODE_GLOBAL) {
VERIFY(fakeif->iff_rx_pp == S_pp);
VERIFY(fakeif->iff_tx_pp == S_pp);
pp_release(fakeif->iff_rx_pp);
fakeif->iff_rx_pp = NULL;
pp_release(fakeif->iff_tx_pp);
fakeif->iff_tx_pp = NULL;
feth_lock();
if (S_pp != NULL && S_pp->pp_refcnt == 1) {
pp_release(S_pp);
S_pp = NULL;
}
feth_unlock();
} else {
if (fakeif->iff_rx_pp != NULL) {
pp_release(fakeif->iff_rx_pp);
fakeif->iff_rx_pp = NULL;
}
if (fakeif->iff_tx_pp != NULL) {
pp_release(fakeif->iff_tx_pp);
fakeif->iff_tx_pp = NULL;
}
}
}
#endif /* SKYWALK */
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s", fakeif->iff_name);
kfree_type(fake_llink, FETH_MAX_LLINKS, fakeif->iff_llink);
kfree_type(struct if_fake, fakeif);
}
static void
feth_release(if_fake_ref fakeif)
{
u_int32_t old_retain_count;
old_retain_count = OSDecrementAtomic(&fakeif->iff_retain_count);
switch (old_retain_count) {
case 0:
VERIFY(old_retain_count != 0);
break;
case 1:
feth_free(fakeif);
break;
default:
break;
}
return;
}
#if SKYWALK
static void
feth_retain(if_fake_ref fakeif)
{
OSIncrementAtomic(&fakeif->iff_retain_count);
}
static void
feth_packet_pool_init_prepare(if_fake_ref fakeif,
struct kern_pbufpool_init *pp_init)
{
uint32_t max_mtu = fakeif->iff_max_mtu;
uint32_t buflet_size = if_fake_buflet_size;
bzero(pp_init, sizeof(*pp_init));
pp_init->kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
pp_init->kbi_flags |= KBIF_VIRTUAL_DEVICE;
pp_init->kbi_packets = 1024; /* TBD configurable */
if (feth_supports_tso(fakeif)) {
buflet_size = if_fake_tso_buffer_size;
}
if (feth_using_multibuflets(fakeif)) {
pp_init->kbi_bufsize = buflet_size;
pp_init->kbi_max_frags = howmany(max_mtu, buflet_size);
pp_init->kbi_buflets = pp_init->kbi_packets *
pp_init->kbi_max_frags;
pp_init->kbi_flags |= KBIF_BUFFER_ON_DEMAND;
} else {
pp_init->kbi_bufsize = max(max_mtu, buflet_size);
pp_init->kbi_max_frags = 1;
pp_init->kbi_buflets = pp_init->kbi_packets;
}
pp_init->kbi_buf_seg_size = skmem_usr_buf_seg_size;
if (if_fake_user_access != 0) {
pp_init->kbi_flags |= KBIF_USER_ACCESS;
}
pp_init->kbi_ctx = NULL;
pp_init->kbi_ctx_retain = NULL;
pp_init->kbi_ctx_release = NULL;
}
static errno_t
feth_packet_pool_make(if_fake_ref fakeif)
{
struct kern_pbufpool_init pp_init;
errno_t err;
feth_packet_pool_init_prepare(fakeif, &pp_init);
switch (fakeif->iff_pp_mode) {
case IFF_PP_MODE_GLOBAL:
feth_lock();
if (S_pp == NULL) {
(void)snprintf((char *)pp_init.kbi_name,
sizeof(pp_init.kbi_name), "%s", "feth shared pp");
err = kern_pbufpool_create(&pp_init, &S_pp, NULL);
}
pp_retain(S_pp);
feth_unlock();
fakeif->iff_rx_pp = S_pp;
pp_retain(S_pp);
fakeif->iff_tx_pp = S_pp;
break;
case IFF_PP_MODE_PRIVATE:
(void)snprintf((char *)pp_init.kbi_name,
sizeof(pp_init.kbi_name), "%s pp", fakeif->iff_name);
err = kern_pbufpool_create(&pp_init, &fakeif->iff_rx_pp, NULL);
pp_retain(fakeif->iff_rx_pp);
fakeif->iff_tx_pp = fakeif->iff_rx_pp;
break;
case IFF_PP_MODE_PRIVATE_SPLIT:
(void)snprintf((char *)pp_init.kbi_name,
sizeof(pp_init.kbi_name), "%s rx pp", fakeif->iff_name);
pp_init.kbi_flags &= ~(KBIF_IODIR_IN | KBIF_IODIR_OUT |
KBIF_BUFFER_ON_DEMAND | KBIF_KERNEL_READONLY);
pp_init.kbi_flags |= (KBIF_IODIR_IN | KBIF_BUFFER_ON_DEMAND);
pp_init.kbi_packets = 1024;
pp_init.kbi_bufsize = if_fake_link_layer_aggregation_factor * 1024;
err = kern_pbufpool_create(&pp_init, &fakeif->iff_rx_pp, NULL);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"rx pp create failed %d", err);
return err;
}
pp_init.kbi_flags &= ~(KBIF_IODIR_IN | KBIF_IODIR_OUT |
KBIF_BUFFER_ON_DEMAND | KBIF_KERNEL_READONLY);
pp_init.kbi_flags |= KBIF_IODIR_OUT;
pp_init.kbi_packets = 1024; /* TBD configurable */
pp_init.kbi_bufsize = fakeif->iff_max_mtu;
(void)snprintf((char *)pp_init.kbi_name,
sizeof(pp_init.kbi_name), "%s tx pp", fakeif->iff_name);
err = kern_pbufpool_create(&pp_init, &fakeif->iff_tx_pp, NULL);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"tx pp create failed %d", err);
pp_release(fakeif->iff_rx_pp);
return err;
}
break;
default:
VERIFY(0);
__builtin_unreachable();
}
return 0;
}
static void
feth_packet_set_trace_tag(kern_packet_t ph, int flag)
{
if (if_fake_trace_tag_flags & flag) {
if (++if_fake_trace_tag_current == 0) {
if_fake_trace_tag_current = 1;
}
kern_packet_set_trace_tag(ph, if_fake_trace_tag_current);
}
}
static errno_t
feth_clone_packet(if_fake_ref dif, kern_packet_t sph, kern_packet_t *pdph)
{
errno_t err = 0;
kern_pbufpool_t pp = dif->iff_rx_pp;
kern_packet_t dph = 0, dph0 = 0;
kern_buflet_t sbuf, dbuf0 = NULL, dbuf;
void *saddr, *daddr;
uint32_t soff, doff;
uint32_t slen, dlen;
uint32_t dlim0, dlim;
sbuf = kern_packet_get_next_buflet(sph, NULL);
saddr = kern_buflet_get_data_address(sbuf);
doff = soff = kern_buflet_get_data_offset(sbuf);
dlen = slen = kern_buflet_get_data_length(sbuf);
/* packet clone is only supported for single-buflet */
ASSERT(kern_packet_get_buflet_count(sph) == 1);
ASSERT(soff == kern_packet_get_headroom(sph));
ASSERT(slen == kern_packet_get_data_length(sph));
dph0 = *pdph;
if (dph0 == 0) {
dlim0 = 0;
} else {
dbuf0 = kern_packet_get_next_buflet(dph0, NULL);
ASSERT(kern_buflet_get_object_limit(dbuf0) ==
PP_BUF_OBJ_SIZE_DEF(pp));
ASSERT(kern_buflet_get_data_limit(dbuf0) % 16 == 0);
dlim0 = ((uintptr_t)kern_buflet_get_object_address(dbuf0) +
kern_buflet_get_object_limit(dbuf0)) -
((uintptr_t)kern_buflet_get_data_address(dbuf0) +
kern_buflet_get_data_limit(dbuf0));
}
if (doff + dlen > dlim0) {
err = kern_pbufpool_alloc_nosleep(pp, 1, &dph);
if (err != 0) {
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_PKT);
return err;
}
dbuf = kern_packet_get_next_buflet(dph, NULL);
ASSERT(kern_buflet_get_data_address(dbuf) ==
kern_buflet_get_object_address(dbuf));
daddr = kern_buflet_get_data_address(dbuf);
dlim = kern_buflet_get_object_limit(dbuf);
ASSERT(dlim == PP_BUF_OBJ_SIZE_DEF(pp));
} else {
err = kern_packet_clone_nosleep(dph0, &dph, KPKT_COPY_LIGHT);
if (err != 0) {
FAKE_LOG(LOG_INFO, FE_DBGF_OUTPUT,
"packet clone err %d", err);
return err;
}
dbuf = kern_packet_get_next_buflet(dph, NULL);
ASSERT(kern_buflet_get_object_address(dbuf) ==
kern_buflet_get_object_address(dbuf0));
daddr = (void *)((uintptr_t)kern_buflet_get_data_address(dbuf0) +
kern_buflet_get_data_limit(dbuf0));
dlim = dlim0;
}
ASSERT(doff + dlen <= dlim);
ASSERT((uintptr_t)daddr % 16 == 0);
bcopy((const void *)((uintptr_t)saddr + soff),
(void *)((uintptr_t)daddr + doff), slen);
dlim = MIN(dlim, P2ROUNDUP(doff + dlen, 16));
err = kern_buflet_set_data_address(dbuf, daddr);
VERIFY(err == 0);
err = kern_buflet_set_data_limit(dbuf, dlim);
VERIFY(err == 0);
err = kern_buflet_set_data_length(dbuf, dlen);
VERIFY(err == 0);
err = kern_buflet_set_data_offset(dbuf, doff);
VERIFY(err == 0);
err = kern_packet_set_headroom(dph, doff);
VERIFY(err == 0);
err = kern_packet_set_link_header_length(dph,
kern_packet_get_link_header_length(sph));
VERIFY(err == 0);
err = kern_packet_set_service_class(dph,
kern_packet_get_service_class(sph));
VERIFY(err == 0);
err = kern_packet_finalize(dph);
VERIFY(err == 0);
*pdph = dph;
return err;
}
static inline void
feth_copy_buflet(kern_buflet_t sbuf, kern_buflet_t dbuf)
{
errno_t err;
uint32_t off, len;
uint8_t *saddr, *daddr;
saddr = kern_buflet_get_data_address(sbuf);
off = kern_buflet_get_data_offset(sbuf);
len = kern_buflet_get_data_length(sbuf);
daddr = kern_buflet_get_data_address(dbuf);
bcopy((saddr + off), (daddr + off), len);
err = kern_buflet_set_data_offset(dbuf, off);
VERIFY(err == 0);
err = kern_buflet_set_data_length(dbuf, len);
VERIFY(err == 0);
}
static int
feth_add_packet_trailer(kern_packet_t ph, void *trailer, size_t trailer_len)
{
errno_t err = 0;
ASSERT(trailer_len <= FETH_TRAILER_LENGTH_MAX);
kern_buflet_t buf = NULL, iter = NULL;
while ((iter = kern_packet_get_next_buflet(ph, iter)) != NULL) {
buf = iter;
}
ASSERT(buf != NULL);
uint32_t dlim = kern_buflet_get_data_limit(buf);
uint32_t doff = kern_buflet_get_data_offset(buf);
uint32_t dlen = kern_buflet_get_data_length(buf);
size_t trailer_room = dlim - doff - dlen;
if (trailer_room < trailer_len) {
FAKE_LOG(LOG_INFO, FE_DBGF_OUTPUT, "not enough room");
return ERANGE;
}
void *data = (void *)((uintptr_t)kern_buflet_get_data_address(buf) + doff + dlen);
memcpy(data, trailer, trailer_len);
err = kern_buflet_set_data_length(buf, dlen + trailer_len);
VERIFY(err == 0);
err = kern_packet_finalize(ph);
VERIFY(err == 0);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%zuB trailer added", trailer_len);
return 0;
}
static int
feth_add_packet_fcs(kern_packet_t ph)
{
uint32_t crc = 0;
int err;
ASSERT(sizeof(crc) == ETHER_CRC_LEN);
kern_buflet_t buf = NULL;
while ((buf = kern_packet_get_next_buflet(ph, buf)) != NULL) {
uint32_t doff = kern_buflet_get_data_offset(buf);
uint32_t dlen = kern_buflet_get_data_length(buf);
void *data = (void *)((uintptr_t)kern_buflet_get_data_address(buf) + doff);
crc = crc32(crc, data, dlen);
}
err = feth_add_packet_trailer(ph, &crc, ETHER_CRC_LEN);
if (!err) {
return err;
}
err = kern_packet_set_link_ethfcs(ph);
VERIFY(err == 0);
return 0;
}
static errno_t
feth_copy_packet(if_fake_ref dif, kern_packet_t sph, kern_packet_t *pdph)
{
errno_t err = 0;
uint16_t i, bufcnt;
mach_vm_address_t baddr;
kern_buflet_t sbuf = NULL, dbuf = NULL;
kern_pbufpool_t pp = dif->iff_rx_pp;
kern_packet_t dph;
boolean_t multi_buflet = feth_using_multibuflets(dif);
bufcnt = kern_packet_get_buflet_count(sph);
ASSERT((bufcnt == 1) || multi_buflet);
*pdph = 0;
err = kern_pbufpool_alloc_nosleep(pp, 1, &dph);
if (err != 0) {
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_PKT);
return err;
}
/* pre-constructed single buflet packet copy */
sbuf = kern_packet_get_next_buflet(sph, NULL);
dbuf = kern_packet_get_next_buflet(dph, NULL);
feth_copy_buflet(sbuf, dbuf);
if (!multi_buflet) {
goto done;
}
/* un-constructed multi-buflet packet copy */
for (i = 1; i < bufcnt; i++) {
kern_buflet_t dbuf_next = NULL;
sbuf = kern_packet_get_next_buflet(sph, sbuf);
VERIFY(sbuf != NULL);
err = kern_pbufpool_alloc_buflet_nosleep(pp, &dbuf_next);
if (err != 0) {
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_BUF);
break;
}
ASSERT(dbuf_next != NULL);
feth_copy_buflet(sbuf, dbuf_next);
err = kern_packet_add_buflet(dph, dbuf, dbuf_next);
VERIFY(err == 0);
dbuf = dbuf_next;
}
if (__improbable(err != 0)) {
dbuf = NULL;
while (i-- != 0) {
dbuf = kern_packet_get_next_buflet(dph, dbuf);
VERIFY(dbuf != NULL);
baddr = (mach_vm_address_t)
kern_buflet_get_data_address(dbuf);
VERIFY(baddr != 0);
}
kern_pbufpool_free(pp, dph);
dph = 0;
}
done:
if (__probable(err == 0)) {
err = kern_packet_set_headroom(dph,
kern_packet_get_headroom(sph));
VERIFY(err == 0);
err = kern_packet_set_link_header_length(dph,
kern_packet_get_link_header_length(sph));
VERIFY(err == 0);
err = kern_packet_set_service_class(dph,
kern_packet_get_service_class(sph));
VERIFY(err == 0);
err = kern_packet_finalize(dph);
VERIFY(err == 0);
VERIFY(bufcnt == kern_packet_get_buflet_count(dph));
*pdph = dph;
}
return err;
}
static inline void
feth_update_pkt_tso_metadata_for_rx(kern_packet_t ph)
{
/*
* Nothing to do if not a TSO offloaded packet.
*/
uint16_t seg_sz = 0;
seg_sz = kern_packet_get_protocol_segment_size(ph);
if (seg_sz == 0) {
return;
}
/*
* For RX, make the packet appear as a fully validated LRO packet.
*/
packet_csum_flags_t csum_flags = PACKET_CSUM_IP_CHECKED |
PACKET_CSUM_IP_VALID | PACKET_CSUM_DATA_VALID |
PACKET_CSUM_PSEUDO_HDR;
(void) kern_packet_set_inet_checksum(ph, csum_flags, 0, 0xFFFF, FALSE);
return;
}
static void
feth_rx_submit(if_fake_ref sif, if_fake_ref dif, kern_packet_t sphs[],
uint32_t n_pkts)
{
errno_t err = 0;
struct kern_channel_ring_stat_increment stats;
kern_channel_ring_t rx_ring = NULL;
kern_channel_slot_t rx_slot = NULL, last_rx_slot = NULL;
kern_packet_t sph = 0, dph = 0;
memset(&stats, 0, sizeof(stats));
rx_ring = dif->iff_rx_ring[0];
if (rx_ring == NULL) {
return;
}
kr_enter(rx_ring, TRUE);
kern_channel_reclaim(rx_ring);
rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);
for (uint32_t i = 0; i < n_pkts && rx_slot != NULL; i++) {
sph = sphs[i];
switch (dif->iff_pp_mode) {
case IFF_PP_MODE_GLOBAL:
sphs[i] = 0;
dph = sph;
feth_update_pkt_tso_metadata_for_rx(dph);
err = kern_packet_finalize(dph);
VERIFY(err == 0);
break;
case IFF_PP_MODE_PRIVATE:
err = feth_copy_packet(dif, sph, &dph);
break;
case IFF_PP_MODE_PRIVATE_SPLIT:
err = feth_clone_packet(dif, sph, &dph);
break;
default:
VERIFY(0);
__builtin_unreachable();
}
if (__improbable(err != 0)) {
continue;
}
if (sif->iff_trailer_length != 0) {
feth_add_packet_trailer(dph, feth_trailer,
sif->iff_trailer_length);
}
if (sif->iff_fcs != 0) {
feth_add_packet_fcs(dph);
}
feth_packet_set_trace_tag(dph, IFF_TT_INPUT);
bpf_tap_packet_in(dif->iff_ifp, DLT_EN10MB, dph, NULL, 0);
stats.kcrsi_slots_transferred++;
stats.kcrsi_bytes_transferred
+= kern_packet_get_data_length(dph);
/* attach the packet to the RX ring */
err = kern_channel_slot_attach_packet(rx_ring, rx_slot, dph);
VERIFY(err == 0);
last_rx_slot = rx_slot;
rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
}
if (last_rx_slot != NULL) {
kern_channel_advance_slot(rx_ring, last_rx_slot);
kern_channel_increment_ring_net_stats(rx_ring, dif->iff_ifp,
&stats);
}
if (rx_ring != NULL) {
kr_exit(rx_ring);
kern_channel_notify(rx_ring, 0);
}
}
static void
feth_rx_queue_submit(if_fake_ref sif, if_fake_ref dif, uint32_t llink_idx,
uint32_t qset_idx, kern_packet_t sphs[], uint32_t n_pkts)
{
errno_t err = 0;
kern_netif_queue_t queue;
kern_packet_t sph = 0, dph = 0;
fake_llink *llink;
fake_qset *qset;
if (llink_idx >= dif->iff_llink_cnt) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
"invalid llink_idx idx %d (max %d) on peer %s",
llink_idx, dif->iff_llink_cnt, dif->iff_name);
return;
}
llink = &dif->iff_llink[llink_idx];
if (qset_idx >= llink->fl_qset_cnt) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
"invalid qset_idx %d (max %d) on peer %s",
qset_idx, llink->fl_qset_cnt, dif->iff_name);
return;
}
qset = &dif->iff_llink[llink_idx].fl_qset[qset_idx];
queue = qset->fqs_rx_queue[0].fq_queue;
if (queue == NULL) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
"NULL default queue (llink_idx %d, qset_idx %d) on peer %s",
llink_idx, qset_idx, dif->iff_name);
return;
}
for (uint32_t i = 0; i < n_pkts; i++) {
uint32_t flags;
sph = sphs[i];
switch (dif->iff_pp_mode) {
case IFF_PP_MODE_GLOBAL:
sphs[i] = 0;
dph = sph;
feth_update_pkt_tso_metadata_for_rx(dph);
break;
case IFF_PP_MODE_PRIVATE:
err = feth_copy_packet(dif, sph, &dph);
break;
case IFF_PP_MODE_PRIVATE_SPLIT:
err = feth_clone_packet(dif, sph, &dph);
break;
default:
VERIFY(0);
__builtin_unreachable();
}
if (__improbable(err != 0)) {
continue;
}
if (sif->iff_trailer_length != 0) {
feth_add_packet_trailer(dph, feth_trailer,
sif->iff_trailer_length);
}
if (sif->iff_fcs != 0) {
feth_add_packet_fcs(dph);
}
feth_packet_set_trace_tag(dph, IFF_TT_INPUT);
bpf_tap_packet_in(dif->iff_ifp, DLT_EN10MB, dph, NULL, 0);
flags = (i == n_pkts - 1) ?
KERN_NETIF_QUEUE_RX_ENQUEUE_FLAG_FLUSH : 0;
kern_netif_queue_rx_enqueue(queue, dph, 1, flags);
}
}
static void
feth_tx_complete(if_fake_ref fakeif, kern_packet_t phs[], uint32_t nphs)
{
for (uint32_t i = 0; i < nphs; i++) {
kern_packet_t ph = phs[i];
if (ph == 0) {
continue;
}
int err = kern_packet_set_tx_completion_status(ph, 0);
VERIFY(err == 0);
kern_packet_tx_completion(ph, fakeif->iff_ifp);
kern_pbufpool_free(fakeif->iff_tx_pp, phs[i]);
phs[i] = 0;
}
}
#define NSEC_PER_USEC 1000ull
/*
* Calculate the time delta that passed from `since' to `until'.
* If `until' happens before `since', returns negative value.
*/
static bool
feth_packet_has_expired(if_fake_ref __unused fakeif, kern_packet_t ph,
uint64_t *out_deadline)
{
uint64_t now;
uint64_t packet_expire_time_mach;
int64_t time_until_expiration;
errno_t err;
bool expired = false;
static mach_timebase_info_data_t clock_timebase = {0, 0};
if (clock_timebase.denom == 0) {
clock_timebase_info(&clock_timebase);
VERIFY(clock_timebase.denom != 0);
}
err = kern_packet_get_expire_time(ph, &packet_expire_time_mach);
if (err) {
goto out;
}
now = mach_absolute_time();
time_until_expiration = packet_expire_time_mach - now;
if (time_until_expiration < 0) {
/* The packet had expired */
expired = true;
goto out;
}
/* Convert the time_delta from mach ticks to nanoseconds */
time_until_expiration *= clock_timebase.numer;
time_until_expiration /= clock_timebase.denom;
/* convert from nanoseconds to microseconds */
time_until_expiration /= 1000ull;
if (if_fake_expiration_threshold_us < time_until_expiration) {
/* packet has some life ahead of it */
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"Packet has %llu usec until expiration",
time_until_expiration);
goto out;
}
out:
if (expired && out_deadline) {
*out_deadline = packet_expire_time_mach;
}
return expired;
}
static errno_t
feth_get_packet_notification_details(if_fake_ref fakeif, kern_packet_t ph,
packet_id_t *pkt_id, uint32_t *nx_port_id)
{
errno_t err = 0;
err = kern_packet_get_packetid(ph, pkt_id);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s err=%d getting packetid", fakeif->iff_name, err);
return err;
}
err = kern_packet_get_tx_nexus_port_id(ph, nx_port_id);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s err=%d getting nx_port_id", fakeif->iff_name, err);
return err;
}
return 0;
}
static packet_expiry_action_t
feth_get_effective_expn_action(if_fake_ref fakeif, kern_packet_t ph)
{
errno_t err;
packet_expiry_action_t expiry_action;
switch (fakeif->iff_tx_exp_policy) {
case IFF_TX_EXP_POLICY_DISABLED:
expiry_action = PACKET_EXPIRY_ACTION_NONE;
break;
case IFF_TX_EXP_POLICY_NOTIFY_ONLY:
expiry_action = PACKET_EXPIRY_ACTION_NOTIFY;
break;
case IFF_TX_EXP_POLICY_DROP_AND_NOTIFY:
expiry_action = PACKET_EXPIRY_ACTION_DROP;
break;
case IFF_TX_EXP_POLICY_METADATA:
err = kern_packet_get_expiry_action(ph, &expiry_action);
if (err != 0) {
if (err != ENOENT) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"Error %d when getting expiry action",
err);
}
expiry_action = PACKET_EXPIRY_ACTION_NONE;
}
break;
default:
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"Unrecognized value %d for \"net.link.fake.tx_exp_policy\"",
fakeif->iff_tx_exp_policy);
expiry_action = PACKET_EXPIRY_ACTION_NONE;
}
return expiry_action;
}
/* returns true if the packet is selected for epxiration and should be dropped */
static bool
feth_tx_expired_error(if_fake_ref fakeif, kern_packet_t ph)
{
int err = 0;
uint32_t nx_port_id = 0;
os_channel_event_packet_transmit_expired_t expn = {0};
packet_expiry_action_t expiry_action = PACKET_EXPIRY_ACTION_NONE;
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC, "%s", fakeif->iff_name);
if (feth_packet_has_expired(fakeif, ph, &expn.packet_tx_expiration_deadline)) {
expiry_action = feth_get_effective_expn_action(fakeif, ph);
}
bool drop_packet = (expiry_action == PACKET_EXPIRY_ACTION_DROP);
if (expiry_action != PACKET_EXPIRY_ACTION_NONE) {
/* set the expiration status code */
expn.packet_tx_expiration_status = drop_packet ?
CHANNEL_EVENT_PKT_TRANSMIT_EXPIRED_ERR_EXPIRED_DROPPED :
CHANNEL_EVENT_PKT_TRANSMIT_EXPIRED_ERR_EXPIRED_NOT_DROPPED;
/* Mark the expiration timestamp */
expn.packet_tx_expiration_timestamp = mach_absolute_time();
err = feth_get_packet_notification_details(fakeif, ph,
&expn.packet_id, &nx_port_id);
if (err == 0) {
err = kern_channel_event_transmit_expired(
fakeif->iff_ifp, &expn, nx_port_id);
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s sent expiry notification on nexus port "
"%u notif code %u",
fakeif->iff_name, nx_port_id,
expn.packet_tx_expiration_status);
}
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s err=%d, nx_port_id: 0x%x",
fakeif->iff_name, err, nx_port_id);
}
}
return drop_packet;
}
/* returns true if the packet is selected for TX error & dropped */
static bool
feth_tx_complete_error(if_fake_ref fakeif, kern_packet_t ph)
{
int err;
if (fakeif->iff_tx_drop_rate == 0 ||
fakeif->iff_tx_pkts_count != fakeif->iff_tx_drop_rate) {
return false;
}
/* simulate TX completion error on the packet */
if (fakeif->iff_tx_completion_mode == IFF_TX_COMPL_MODE_SYNC) {
err = kern_packet_set_tx_completion_status(ph,
CHANNEL_EVENT_PKT_TRANSMIT_STATUS_ERR_RETRY_FAILED);
VERIFY(err == 0);
kern_packet_tx_completion(ph, fakeif->iff_ifp);
} else {
uint32_t nx_port_id = 0;
os_channel_event_packet_transmit_status_t pkt_tx_status = {0};
pkt_tx_status.packet_status =
CHANNEL_EVENT_PKT_TRANSMIT_STATUS_ERR_RETRY_FAILED;
err = feth_get_packet_notification_details(fakeif, ph,
&pkt_tx_status.packet_id, &nx_port_id);
if (err == 0) {
err = kern_channel_event_transmit_status(
fakeif->iff_ifp, &pkt_tx_status, nx_port_id);
}
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s err=%d, nx_port_id: 0x%x",
fakeif->iff_name, err, nx_port_id);
}
}
return true;
}
static void
feth_if_adv(thread_call_param_t arg0, thread_call_param_t arg1)
{
#pragma unused(arg1)
errno_t error;
if_fake_ref fakeif = (if_fake_ref)arg0;
struct ifnet_interface_advisory if_adv;
struct ifnet_stats_param if_stat;
feth_lock();
if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
feth_unlock();
return;
}
feth_unlock();
if (!fakeif->iff_intf_adv_enabled) {
goto done;
}
error = ifnet_stat(fakeif->iff_ifp, &if_stat);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, 0, "%s: ifnet_stat() failed %d",
fakeif->iff_name, error);
goto done;
}
if_adv.header.version = IF_INTERFACE_ADVISORY_VERSION_CURRENT;
if_adv.header.direction = IF_INTERFACE_ADVISORY_DIRECTION_TX;
if_adv.header.interface_type =
IF_INTERFACE_ADVISORY_INTERFACE_TYPE_WIFI;
if_adv.capacity.timestamp = mach_absolute_time();
if_adv.capacity.rate_trend_suggestion =
IF_INTERFACE_ADVISORY_RATE_SUGGESTION_RAMP_NEUTRAL;
if_adv.capacity.max_bandwidth = 1000 * 1000 * 1000; /* 1Gbps */
if_adv.capacity.total_byte_count = if_stat.packets_out;
if_adv.capacity.average_throughput = 1000 * 1000 * 1000; /* 1Gbps */
if_adv.capacity.flushable_queue_size = UINT32_MAX;
if_adv.capacity.non_flushable_queue_size = UINT32_MAX;
if_adv.capacity.average_delay = 1; /* ms */
error = fakeif->iff_intf_adv_notify(fakeif->iff_intf_adv_kern_ctx,
&if_adv);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, 0,
"%s: interface advisory report failed %d",
fakeif->iff_name, error);
}
done:
feth_lock();
if (!feth_is_detaching(fakeif) && fakeif->iff_channel_connected) {
uint64_t deadline;
clock_interval_to_deadline(fakeif->iff_adv_interval,
NSEC_PER_MSEC, &deadline);
thread_call_enter_delayed(fakeif->iff_if_adv_tcall, deadline);
}
feth_unlock();
}
static int
feth_if_adv_tcall_create(if_fake_ref fakeif)
{
uint64_t deadline;
feth_lock();
ASSERT(fakeif->iff_if_adv_tcall == NULL);
ASSERT(fakeif->iff_adv_interval > 0);
ASSERT(fakeif->iff_channel_connected);
fakeif->iff_if_adv_tcall =
thread_call_allocate_with_options(feth_if_adv,
(thread_call_param_t)fakeif, THREAD_CALL_PRIORITY_KERNEL,
THREAD_CALL_OPTIONS_ONCE);
if (fakeif->iff_if_adv_tcall == NULL) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s if_adv tcall alloc failed",
fakeif->iff_name);
return ENXIO;
}
/* retain for the interface advisory thread call */
feth_retain(fakeif);
clock_interval_to_deadline(fakeif->iff_adv_interval,
NSEC_PER_MSEC, &deadline);
thread_call_enter_delayed(fakeif->iff_if_adv_tcall, deadline);
feth_unlock();
return 0;
}
static void
feth_if_adv_tcall_destroy(if_fake_ref fakeif)
{
thread_call_t tcall;
feth_lock();
ASSERT(fakeif->iff_if_adv_tcall != NULL);
tcall = fakeif->iff_if_adv_tcall;
feth_unlock();
(void) thread_call_cancel_wait(tcall);
if (!thread_call_free(tcall)) {
boolean_t freed;
(void) thread_call_cancel_wait(tcall);
freed = thread_call_free(tcall);
VERIFY(freed);
}
feth_lock();
fakeif->iff_if_adv_tcall = NULL;
feth_unlock();
/* release for the interface advisory thread call */
feth_release(fakeif);
}
/**
** nexus netif domain provider
**/
static errno_t
feth_nxdp_init(kern_nexus_domain_provider_t domprov)
{
#pragma unused(domprov)
return 0;
}
static void
feth_nxdp_fini(kern_nexus_domain_provider_t domprov)
{
#pragma unused(domprov)
}
static uuid_t feth_nx_dom_prov;
static errno_t
feth_register_nexus_domain_provider(void)
{
const struct kern_nexus_domain_provider_init dp_init = {
.nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
.nxdpi_flags = 0,
.nxdpi_init = feth_nxdp_init,
.nxdpi_fini = feth_nxdp_fini
};
errno_t err = 0;
/* feth_nxdp_init() is called before this function returns */
err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
(const uint8_t *)
"com.apple.feth",
&dp_init, sizeof(dp_init),
&feth_nx_dom_prov);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"failed to register domain provider");
return err;
}
return 0;
}
/**
** netif nexus routines
**/
static if_fake_ref
feth_nexus_context(kern_nexus_t nexus)
{
if_fake_ref fakeif;
fakeif = (if_fake_ref)kern_nexus_get_context(nexus);
assert(fakeif != NULL);
return fakeif;
}
static uint8_t
feth_find_tx_ring_by_svc(kern_packet_svc_class_t svc_class)
{
switch (svc_class) {
case KPKT_SC_VO:
return 0;
case KPKT_SC_VI:
return 1;
case KPKT_SC_BE:
return 2;
case KPKT_SC_BK:
return 3;
default:
VERIFY(0);
return 0;
}
}
static errno_t
feth_nx_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
void **ring_ctx)
{
if_fake_ref fakeif;
int err;
#pragma unused(nxprov, channel, ring_ctx)
feth_lock();
fakeif = feth_nexus_context(nexus);
if (feth_is_detaching(fakeif)) {
feth_unlock();
return 0;
}
if (is_tx_ring) {
if (feth_in_wmm_mode(fakeif)) {
kern_packet_svc_class_t svc_class;
uint8_t ring_idx;
err = kern_channel_get_service_class(ring, &svc_class);
VERIFY(err == 0);
ring_idx = feth_find_tx_ring_by_svc(svc_class);
VERIFY(ring_idx < IFF_NUM_TX_RINGS_WMM_MODE);
VERIFY(fakeif->iff_tx_ring[ring_idx] == NULL);
fakeif->iff_tx_ring[ring_idx] = ring;
} else {
VERIFY(fakeif->iff_tx_ring[0] == NULL);
fakeif->iff_tx_ring[0] = ring;
}
} else {
VERIFY(fakeif->iff_rx_ring[0] == NULL);
fakeif->iff_rx_ring[0] = ring;
}
fakeif->iff_nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
feth_unlock();
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: %s ring init",
fakeif->iff_name, is_tx_ring ? "TX" : "RX");
return 0;
}
static void
feth_nx_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
kern_channel_ring_t ring)
{
#pragma unused(nxprov, ring)
if_fake_ref fakeif;
thread_call_t tcall = NULL;
feth_lock();
fakeif = feth_nexus_context(nexus);
if (fakeif->iff_rx_ring[0] == ring) {
fakeif->iff_rx_ring[0] = NULL;
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: RX ring fini", fakeif->iff_name);
} else if (feth_in_wmm_mode(fakeif)) {
int i;
for (i = 0; i < IFF_MAX_TX_RINGS; i++) {
if (fakeif->iff_tx_ring[i] == ring) {
fakeif->iff_tx_ring[i] = NULL;
break;
}
}
for (i = 0; i < IFF_MAX_TX_RINGS; i++) {
if (fakeif->iff_tx_ring[i] != NULL) {
break;
}
}
if (i == IFF_MAX_TX_RINGS) {
tcall = fakeif->iff_doorbell_tcall;
fakeif->iff_doorbell_tcall = NULL;
}
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: TX ring fini", fakeif->iff_name);
} else if (fakeif->iff_tx_ring[0] == ring) {
tcall = fakeif->iff_doorbell_tcall;
fakeif->iff_doorbell_tcall = NULL;
fakeif->iff_tx_ring[0] = NULL;
}
fakeif->iff_nifs = NULL;
feth_unlock();
if (tcall != NULL) {
boolean_t success;
success = thread_call_cancel_wait(tcall);
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: thread_call_cancel %s", fakeif->iff_name,
success ? "SUCCESS" : "FAILURE");
if (!success) {
feth_lock();
if (fakeif->iff_doorbell_tcall_active) {
fakeif->iff_waiting_for_tcall = TRUE;
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: *waiting for threadcall",
fakeif->iff_name);
do {
msleep(fakeif, &feth_lck_mtx,
PZERO, "feth threadcall", 0);
} while (fakeif->iff_doorbell_tcall_active);
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: ^threadcall done",
fakeif->iff_name);
fakeif->iff_waiting_for_tcall = FALSE;
}
feth_unlock();
}
success = thread_call_free(tcall);
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: thread_call_free %s",
fakeif->iff_name,
success ? "SUCCESS" : "FAILURE");
feth_release(fakeif);
VERIFY(success == TRUE);
}
}
static errno_t
feth_nx_pre_connect(kern_nexus_provider_t nxprov,
proc_t proc, kern_nexus_t nexus, nexus_port_t port, kern_channel_t channel,
void **channel_context)
{
#pragma unused(nxprov, proc, nexus, port, channel, channel_context)
return 0;
}
static errno_t
feth_nx_connected(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
int err;
if_fake_ref fakeif;
fakeif = feth_nexus_context(nexus);
feth_lock();
if (feth_is_detaching(fakeif)) {
feth_unlock();
return EBUSY;
}
feth_retain(fakeif);
fakeif->iff_channel_connected = TRUE;
feth_unlock();
if (feth_has_intf_advisory_configured(fakeif)) {
err = feth_if_adv_tcall_create(fakeif);
if (err != 0) {
return err;
}
}
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: connected channel %p",
fakeif->iff_name, channel);
return 0;
}
static void
feth_nx_pre_disconnect(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
if_fake_ref fakeif;
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: pre-disconnect channel %p",
fakeif->iff_name, channel);
/* Quiesce the interface and flush any pending outbound packets. */
if_down(fakeif->iff_ifp);
feth_lock();
fakeif->iff_channel_connected = FALSE;
feth_unlock();
if (fakeif->iff_if_adv_tcall != NULL) {
feth_if_adv_tcall_destroy(fakeif);
}
}
static void
feth_nx_disconnected(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
if_fake_ref fakeif;
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: disconnected channel %p",
fakeif->iff_name, channel);
feth_release(fakeif);
}
static errno_t
feth_nx_slot_init(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_ring_t ring, kern_channel_slot_t slot,
uint32_t slot_index, struct kern_slot_prop **slot_prop_addr,
void **slot_context)
{
#pragma unused(nxprov, nexus, ring, slot, slot_index, slot_prop_addr, slot_context)
return 0;
}
static void
feth_nx_slot_fini(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_ring_t ring, kern_channel_slot_t slot,
uint32_t slot_index)
{
#pragma unused(nxprov, nexus, ring, slot, slot_index)
}
static errno_t
feth_nx_sync_tx(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_ring_t tx_ring, uint32_t flags)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
ifnet_t ifp;
kern_channel_slot_t last_tx_slot = NULL;
ifnet_t peer_ifp;
if_fake_ref peer_fakeif = NULL;
struct kern_channel_ring_stat_increment stats;
kern_channel_slot_t tx_slot;
struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
kern_packet_t pkts[IFF_MAX_BATCH_SIZE];
uint32_t n_pkts = 0;
memset(&stats, 0, sizeof(stats));
STATS_INC(nifs, NETIF_STATS_TX_SYNC);
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s ring %d flags 0x%x", fakeif->iff_name,
tx_ring->ckr_ring_id, flags);
(void)flags;
feth_lock();
if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
feth_unlock();
return 0;
}
ifp = fakeif->iff_ifp;
peer_ifp = fakeif->iff_peer;
if (peer_ifp != NULL) {
peer_fakeif = ifnet_get_if_fake(peer_ifp);
if (peer_fakeif != NULL) {
if (feth_is_detaching(peer_fakeif)) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s peer fakeif %s is detaching",
fakeif->iff_name, peer_fakeif->iff_name);
goto done;
}
if (!peer_fakeif->iff_channel_connected) {
if (fakeif->iff_tx_exp_policy ==
IFF_TX_EXP_POLICY_DISABLED) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s peer fakeif %s channel not connected, expn: %d",
fakeif->iff_name, peer_fakeif->iff_name,
fakeif->iff_tx_exp_policy);
goto done;
}
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s no peer fakeif (peer %p)",
fakeif->iff_name, peer_ifp);
goto done;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s no peer", fakeif->iff_name);
goto done;
}
tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
while (tx_slot != NULL) {
uint16_t off;
kern_packet_t sph;
/* detach the packet from the TX ring */
sph = kern_channel_slot_get_packet(tx_ring, tx_slot);
VERIFY(sph != 0);
kern_channel_slot_detach_packet(tx_ring, tx_slot, sph);
/* bpf tap output */
off = kern_packet_get_headroom(sph);
VERIFY(off >= fakeif->iff_tx_headroom);
kern_packet_set_link_header_length(sph, ETHER_HDR_LEN);
feth_packet_set_trace_tag(sph, IFF_TT_OUTPUT);
bpf_tap_packet_out(ifp, DLT_EN10MB, sph, NULL, 0);
/* drop packets, if requested */
fakeif->iff_tx_pkts_count++;
if (feth_tx_expired_error(fakeif, sph) ||
feth_tx_complete_error(fakeif, sph) ||
!peer_fakeif->iff_channel_connected) {
fakeif->iff_tx_pkts_count = 0;
kern_pbufpool_free(fakeif->iff_tx_pp, sph);
STATS_INC(nifs, NETIF_STATS_DROP);
goto next_tx_slot;
}
ASSERT(sph != 0);
STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
stats.kcrsi_slots_transferred++;
stats.kcrsi_bytes_transferred
+= kern_packet_get_data_length(sph);
/* prepare batch for receiver */
pkts[n_pkts++] = sph;
if (n_pkts == IFF_MAX_BATCH_SIZE) {
feth_rx_submit(fakeif, peer_fakeif, pkts, n_pkts);
feth_tx_complete(fakeif, pkts, n_pkts);
n_pkts = 0;
}
next_tx_slot:
last_tx_slot = tx_slot;
tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
}
/* catch last batch for receiver */
if (n_pkts != 0) {
feth_rx_submit(fakeif, peer_fakeif, pkts, n_pkts);
feth_tx_complete(fakeif, pkts, n_pkts);
n_pkts = 0;
}
if (last_tx_slot != NULL) {
kern_channel_advance_slot(tx_ring, last_tx_slot);
kern_channel_increment_ring_net_stats(tx_ring, ifp, &stats);
}
done:
feth_unlock();
return 0;
}
static errno_t
feth_nx_sync_rx(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_ring_t ring, uint32_t flags)
{
#pragma unused(nxprov, ring, flags)
if_fake_ref fakeif;
struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
STATS_INC(nifs, NETIF_STATS_RX_SYNC);
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT, "%s", fakeif->iff_name);
return 0;
}
static errno_t
feth_nx_tx_dequeue_driver_managed(if_fake_ref fakeif, boolean_t doorbell_ctxt)
{
int i;
errno_t error = 0;
boolean_t more;
for (i = 0; i < IFF_NUM_TX_RINGS_WMM_MODE; i++) {
kern_channel_ring_t ring = fakeif->iff_tx_ring[i];
if (ring != NULL) {
error = kern_channel_tx_refill(ring, UINT32_MAX,
UINT32_MAX, doorbell_ctxt, &more);
}
if (error != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: TX refill ring %d (%s) %d",
fakeif->iff_name, ring->ckr_ring_id,
doorbell_ctxt ? "sync" : "async", error);
if (!((error == EAGAIN) || (error == EBUSY))) {
break;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: TX refilled ring %d (%s)",
fakeif->iff_name, ring->ckr_ring_id,
doorbell_ctxt ? "sync" : "async");
}
}
return error;
}
static void
feth_async_doorbell(thread_call_param_t arg0, thread_call_param_t arg1)
{
#pragma unused(arg1)
errno_t error;
if_fake_ref fakeif = (if_fake_ref)arg0;
kern_channel_ring_t ring;
boolean_t more;
feth_lock();
ring = fakeif->iff_tx_ring[0];
if (feth_is_detaching(fakeif) ||
!fakeif->iff_channel_connected ||
ring == NULL) {
goto done;
}
fakeif->iff_doorbell_tcall_active = TRUE;
feth_unlock();
if (feth_in_wmm_mode(fakeif)) {
error = feth_nx_tx_dequeue_driver_managed(fakeif, FALSE);
} else {
error = kern_channel_tx_refill(ring, UINT32_MAX,
UINT32_MAX, FALSE, &more);
}
if (error != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s: TX refill failed %d",
fakeif->iff_name, error);
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s: TX refilled",
fakeif->iff_name);
}
feth_lock();
done:
fakeif->iff_doorbell_tcall_active = FALSE;
if (fakeif->iff_waiting_for_tcall) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: threadcall waking up waiter", fakeif->iff_name);
wakeup((caddr_t)fakeif);
}
feth_unlock();
}
static void
feth_schedule_async_doorbell(if_fake_ref fakeif)
{
thread_call_t tcall;
feth_lock();
if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
feth_unlock();
return;
}
tcall = fakeif->iff_doorbell_tcall;
if (tcall != NULL) {
thread_call_enter(tcall);
} else {
tcall = thread_call_allocate_with_options(feth_async_doorbell,
(thread_call_param_t)fakeif,
THREAD_CALL_PRIORITY_KERNEL,
THREAD_CALL_OPTIONS_ONCE);
if (tcall == NULL) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT,
"%s tcall alloc failed", fakeif->iff_name);
} else {
fakeif->iff_doorbell_tcall = tcall;
feth_retain(fakeif);
thread_call_enter(tcall);
}
}
feth_unlock();
}
static errno_t
feth_nx_tx_doorbell(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, kern_channel_ring_t ring, uint32_t flags)
{
#pragma unused(nxprov, ring, flags)
errno_t error;
if_fake_ref fakeif;
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s", fakeif->iff_name);
if ((flags & KERN_NEXUS_TXDOORBELLF_ASYNC_REFILL) == 0) {
boolean_t more;
/* synchronous tx refill */
if (feth_in_wmm_mode(fakeif)) {
error = feth_nx_tx_dequeue_driver_managed(fakeif, TRUE);
} else {
error = kern_channel_tx_refill(ring, UINT32_MAX,
UINT32_MAX, TRUE, &more);
}
if (error != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: TX refill (sync) %d", fakeif->iff_name, error);
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: TX refilled (sync)", fakeif->iff_name);
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: schedule async refill", fakeif->iff_name);
feth_schedule_async_doorbell(fakeif);
}
return 0;
}
static errno_t
feth_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
{
if_fake_ref fakeif;
fakeif = (if_fake_ref)kern_nexus_get_context(nexus);
feth_ifnet_set_attrs(fakeif, ifp);
return 0;
}
static errno_t
feth_nx_intf_adv_config(void *prov_ctx, bool enable)
{
if_fake_ref fakeif = prov_ctx;
feth_lock();
fakeif->iff_intf_adv_enabled = enable;
feth_unlock();
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s enable %d", fakeif->iff_name, enable);
return 0;
}
static errno_t
fill_capab_interface_advisory(if_fake_ref fakeif, void *contents, uint32_t *len)
{
struct kern_nexus_capab_interface_advisory *capab = contents;
if (*len != sizeof(*capab)) {
return EINVAL;
}
if (capab->kncia_version !=
KERN_NEXUS_CAPAB_INTERFACE_ADVISORY_VERSION_1) {
return EINVAL;
}
if (!feth_has_intf_advisory_configured(fakeif)) {
return ENOTSUP;
}
VERIFY(capab->kncia_notify != NULL);
fakeif->iff_intf_adv_kern_ctx = capab->kncia_kern_context;
fakeif->iff_intf_adv_notify = capab->kncia_notify;
capab->kncia_provider_context = fakeif;
capab->kncia_config = feth_nx_intf_adv_config;
return 0;
}
static errno_t
feth_notify_steering_info(void *prov_ctx, void *qset_ctx,
struct ifnet_traffic_descriptor_common *td, bool add)
{
#pragma unused(td)
if_fake_ref fakeif = prov_ctx;
fake_qset *qset = qset_ctx;
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s: notify_steering_info: qset_id 0x%llx, %s",
fakeif->iff_name, qset->fqs_id, add ? "add" : "remove");
return 0;
}
static errno_t
fill_capab_qset_extensions(if_fake_ref fakeif, void *contents, uint32_t *len)
{
struct kern_nexus_capab_qset_extensions *capab = contents;
if (*len != sizeof(*capab)) {
return EINVAL;
}
if (capab->cqe_version !=
KERN_NEXUS_CAPAB_QSET_EXTENSIONS_VERSION_1) {
return EINVAL;
}
capab->cqe_prov_ctx = fakeif;
capab->cqe_notify_steering_info = feth_notify_steering_info;
return 0;
}
static errno_t
feth_nx_capab_config(kern_nexus_provider_t nxprov, kern_nexus_t nx,
kern_nexus_capab_t capab, void *contents, uint32_t *len)
{
#pragma unused(nxprov)
errno_t error;
if_fake_ref fakeif;
fakeif = feth_nexus_context(nx);
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL, "%s", fakeif->iff_name);
switch (capab) {
case KERN_NEXUS_CAPAB_INTERFACE_ADVISORY:
error = fill_capab_interface_advisory(fakeif, contents, len);
break;
case KERN_NEXUS_CAPAB_QSET_EXTENSIONS:
error = fill_capab_qset_extensions(fakeif, contents, len);
break;
default:
error = ENOTSUP;
break;
}
return error;
}
static int
feth_set_tso_mtu(ifnet_t ifp, uint32_t tso_v4_mtu, uint32_t tso_v6_mtu)
{
int error;
error = ifnet_set_tso_mtu(ifp, AF_INET, tso_v4_mtu);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
"set TSO MTU IPv4 failed on %s, err %d",
if_name(ifp), error);
return error;
}
error = ifnet_set_tso_mtu(ifp, AF_INET6, tso_v6_mtu);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
"set TSO MTU IPv6 failed on %s, err %d",
if_name(ifp), error);
return error;
}
return 0;
}
static int
feth_set_tso_offload(ifnet_t ifp)
{
ifnet_offload_t offload;
int error;
offload = IFNET_TSO_IPV4 | IFNET_TSO_IPV6;
error = ifnet_set_offload(ifp, offload);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
"set TSO offload failed on %s, err %d",
if_name(ifp), error);
goto done;
}
error = feth_set_tso_mtu(ifp, if_fake_tso_buffer_size,
if_fake_tso_buffer_size);
done:
return error;
}
static errno_t
create_netif_provider_and_instance(if_fake_ref fakeif,
struct ifnet_init_eparams * init_params, ifnet_t *ifp,
uuid_t * provider, uuid_t * instance)
{
errno_t err;
nexus_controller_t controller = kern_nexus_shared_controller();
struct kern_nexus_net_init net_init;
nexus_name_t provider_name;
nexus_attr_t nexus_attr = NULL;
struct kern_nexus_provider_init prov_init = {
.nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
.nxpi_flags = NXPIF_VIRTUAL_DEVICE,
.nxpi_pre_connect = feth_nx_pre_connect,
.nxpi_connected = feth_nx_connected,
.nxpi_pre_disconnect = feth_nx_pre_disconnect,
.nxpi_disconnected = feth_nx_disconnected,
.nxpi_ring_init = feth_nx_ring_init,
.nxpi_ring_fini = feth_nx_ring_fini,
.nxpi_slot_init = feth_nx_slot_init,
.nxpi_slot_fini = feth_nx_slot_fini,
.nxpi_sync_tx = feth_nx_sync_tx,
.nxpi_sync_rx = feth_nx_sync_rx,
.nxpi_tx_doorbell = feth_nx_tx_doorbell,
.nxpi_config_capab = feth_nx_capab_config,
};
_CASSERT(IFF_MAX_RX_RINGS == 1);
err = kern_nexus_attr_create(&nexus_attr);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"nexus attribute creation failed, error %d", err);
goto failed;
}
if (feth_in_wmm_mode(fakeif)) {
err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_TX_RINGS,
IFF_NUM_TX_RINGS_WMM_MODE);
VERIFY(err == 0);
err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_RX_RINGS,
IFF_NUM_RX_RINGS_WMM_MODE);
VERIFY(err == 0);
err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_QMAP,
NEXUS_QMAP_TYPE_WMM);
VERIFY(err == 0);
}
err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_ANONYMOUS, 1);
VERIFY(err == 0);
snprintf((char *)provider_name, sizeof(provider_name),
"com.apple.netif.%s", fakeif->iff_name);
err = kern_nexus_controller_register_provider(controller,
feth_nx_dom_prov,
provider_name,
&prov_init,
sizeof(prov_init),
nexus_attr,
provider);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"register provider failed, error %d", err);
goto failed;
}
bzero(&net_init, sizeof(net_init));
net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
net_init.nxneti_flags = 0;
net_init.nxneti_eparams = init_params;
net_init.nxneti_lladdr = NULL;
net_init.nxneti_prepare = feth_netif_prepare;
net_init.nxneti_rx_pbufpool = fakeif->iff_rx_pp;
net_init.nxneti_tx_pbufpool = fakeif->iff_tx_pp;
err = kern_nexus_controller_alloc_net_provider_instance(controller,
*provider,
fakeif,
NULL,
instance,
&net_init,
ifp);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"alloc_net_provider_instance failed, %d", err);
kern_nexus_controller_deregister_provider(controller,
*provider);
uuid_clear(*provider);
goto failed;
}
if (feth_supports_tso(fakeif)) {
if ((err = feth_set_tso_offload(*ifp)) != 0) {
goto failed;
}
}
failed:
if (nexus_attr != NULL) {
kern_nexus_attr_destroy(nexus_attr);
}
return err;
}
/*
* The nif_stats need to be referenced because we don't want it set
* to NULL until the last llink is removed.
*/
static void
get_nexus_stats(if_fake_ref fakeif, kern_nexus_t nexus)
{
if (++fakeif->iff_nifs_ref == 1) {
ASSERT(fakeif->iff_nifs == NULL);
fakeif->iff_nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
}
}
static void
clear_nexus_stats(if_fake_ref fakeif)
{
if (--fakeif->iff_nifs_ref == 0) {
ASSERT(fakeif->iff_nifs != NULL);
fakeif->iff_nifs = NULL;
}
}
static errno_t
feth_nx_qset_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
void *llink_ctx, uint8_t qset_idx, uint64_t qset_id, kern_netif_qset_t qset,
void **qset_ctx)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
fake_llink *fl = llink_ctx;
fake_qset *fqs;
feth_lock();
fakeif = feth_nexus_context(nexus);
if (feth_is_detaching(fakeif)) {
feth_unlock();
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"%s: detaching", fakeif->iff_name);
return ENXIO;
}
if (qset_idx >= fl->fl_qset_cnt) {
feth_unlock();
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"%s: invalid qset_idx %d", fakeif->iff_name, qset_idx);
return EINVAL;
}
fqs = &fl->fl_qset[qset_idx];
ASSERT(fqs->fqs_qset == NULL);
fqs->fqs_qset = qset;
fqs->fqs_id = qset_id;
*qset_ctx = fqs;
/* XXX This should really be done during registration */
get_nexus_stats(fakeif, nexus);
feth_unlock();
return 0;
}
static void
feth_nx_qset_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
void *qset_ctx)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
fake_qset *fqs = qset_ctx;
feth_lock();
fakeif = feth_nexus_context(nexus);
clear_nexus_stats(fakeif);
ASSERT(fqs->fqs_qset != NULL);
fqs->fqs_qset = NULL;
fqs->fqs_id = 0;
feth_unlock();
}
static errno_t
feth_nx_queue_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
void *qset_ctx, uint8_t qidx, bool tx, kern_netif_queue_t queue,
void **queue_ctx)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
fake_qset *fqs = qset_ctx;
fake_queue *fq;
feth_lock();
fakeif = feth_nexus_context(nexus);
if (feth_is_detaching(fakeif)) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"%s: detaching", fakeif->iff_name);
feth_unlock();
return ENXIO;
}
if (tx) {
if (qidx >= fqs->fqs_tx_queue_cnt) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"%s: invalid tx qidx %d", fakeif->iff_name, qidx);
feth_unlock();
return EINVAL;
}
fq = &fqs->fqs_tx_queue[qidx];
} else {
if (qidx >= fqs->fqs_rx_queue_cnt) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
"%s: invalid rx qidx %d", fakeif->iff_name, qidx);
feth_unlock();
return EINVAL;
}
fq = &fqs->fqs_rx_queue[qidx];
}
ASSERT(fq->fq_queue == NULL);
fq->fq_queue = queue;
*queue_ctx = fq;
feth_unlock();
return 0;
}
static void
feth_nx_queue_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
void *queue_ctx)
{
#pragma unused(nxprov, nexus)
fake_queue *fq = queue_ctx;
feth_lock();
ASSERT(fq->fq_queue != NULL);
fq->fq_queue = NULL;
feth_unlock();
}
static void
feth_nx_tx_queue_deliver_pkt_chain(if_fake_ref fakeif, kern_packet_t sph,
struct netif_stats *nifs, if_fake_ref peer_fakeif,
uint32_t llink_idx, uint32_t qset_idx)
{
kern_packet_t pkts[IFF_MAX_BATCH_SIZE];
uint32_t n_pkts = 0;
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s -> %s", fakeif->iff_name, peer_fakeif->iff_name);
while (sph != 0) {
uint16_t off;
kern_packet_t next;
next = kern_packet_get_next(sph);
kern_packet_set_next(sph, 0);
/* bpf tap output */
off = kern_packet_get_headroom(sph);
VERIFY(off >= fakeif->iff_tx_headroom);
kern_packet_set_link_header_length(sph, ETHER_HDR_LEN);
feth_packet_set_trace_tag(sph, IFF_TT_OUTPUT);
bpf_tap_packet_out(fakeif->iff_ifp, DLT_EN10MB, sph, NULL, 0);
/* drop packets, if requested */
fakeif->iff_tx_pkts_count++;
if (feth_tx_expired_error(fakeif, sph) ||
feth_tx_complete_error(fakeif, sph)) {
fakeif->iff_tx_pkts_count = 0;
kern_pbufpool_free(fakeif->iff_tx_pp, sph);
STATS_INC(nifs, NETIF_STATS_DROP);
goto next_pkt;
}
ASSERT(sph != 0);
STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
STATS_INC(nifs, NETIF_STATS_TX_PACKETS);
/* prepare batch for receiver */
pkts[n_pkts++] = sph;
if (n_pkts == IFF_MAX_BATCH_SIZE) {
feth_rx_queue_submit(fakeif, peer_fakeif, llink_idx,
qset_idx, pkts, n_pkts);
feth_tx_complete(fakeif, pkts, n_pkts);
n_pkts = 0;
}
next_pkt:
sph = next;
}
/* catch last batch for receiver */
if (n_pkts != 0) {
feth_rx_queue_submit(fakeif, peer_fakeif, llink_idx, qset_idx,
pkts, n_pkts);
feth_tx_complete(fakeif, pkts, n_pkts);
n_pkts = 0;
}
}
static errno_t
feth_nx_tx_qset_notify(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
void *qset_ctx, uint32_t flags)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
ifnet_t ifp;
ifnet_t peer_ifp;
if_fake_ref peer_fakeif = NULL;
struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
fake_qset *qset = qset_ctx;
boolean_t detaching, connected;
uint32_t i;
errno_t err;
STATS_INC(nifs, NETIF_STATS_TX_SYNC);
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s qset %p, idx %d, flags 0x%x", fakeif->iff_name, qset,
qset->fqs_idx, flags);
feth_lock();
detaching = feth_is_detaching(fakeif);
connected = fakeif->iff_channel_connected;
if (detaching || !connected) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: detaching %s, channel connected %s",
fakeif->iff_name,
(detaching ? "true" : "false"),
(connected ? "true" : "false"));
feth_unlock();
return 0;
}
ifp = fakeif->iff_ifp;
peer_ifp = fakeif->iff_peer;
if (peer_ifp != NULL) {
peer_fakeif = ifnet_get_if_fake(peer_ifp);
if (peer_fakeif != NULL) {
detaching = feth_is_detaching(peer_fakeif);
connected = peer_fakeif->iff_channel_connected;
if (detaching || !connected) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"peer %s: detaching %s, "
"channel connected %s",
peer_fakeif->iff_name,
(detaching ? "true" : "false"),
(connected ? "true" : "false"));
goto done;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"peer_fakeif is NULL");
goto done;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "peer_ifp is NULL");
goto done;
}
if (if_fake_switch_combined_mode &&
qset->fqs_dequeue_cnt >= if_fake_switch_mode_frequency) {
if (qset->fqs_combined_mode) {
kern_netif_set_qset_separate(qset->fqs_qset);
} else {
kern_netif_set_qset_combined(qset->fqs_qset);
}
qset->fqs_combined_mode = !qset->fqs_combined_mode;
qset->fqs_dequeue_cnt = 0;
}
for (i = 0; i < qset->fqs_tx_queue_cnt; i++) {
kern_packet_t sph = 0;
kern_netif_queue_t queue = qset->fqs_tx_queue[i].fq_queue;
boolean_t more = FALSE;
err = kern_netif_queue_tx_dequeue(queue, UINT32_MAX, UINT32_MAX,
&more, &sph);
if (err != 0 && err != EAGAIN) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s queue %p dequeue failed: err "
"%d", fakeif->iff_name, queue, err);
}
feth_nx_tx_queue_deliver_pkt_chain(fakeif, sph, nifs,
peer_fakeif, qset->fqs_llink_idx, qset->fqs_idx);
}
done:
feth_unlock();
return 0;
}
static errno_t
feth_nx_queue_tx_push(kern_nexus_provider_t nxprov,
kern_nexus_t nexus, void *queue_ctx, kern_packet_t *ph,
uint32_t *packetCount, uint32_t *byteCount)
{
#pragma unused(nxprov)
if_fake_ref fakeif;
ifnet_t ifp;
ifnet_t peer_ifp;
if_fake_ref peer_fakeif = NULL;
struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
fake_queue *fq = queue_ctx;
boolean_t detaching, connected;
STATS_INC(nifs, NETIF_STATS_TX_SYNC);
fakeif = feth_nexus_context(nexus);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s queue %p", fakeif->iff_name, fq);
feth_lock();
detaching = feth_is_detaching(fakeif);
connected = fakeif->iff_channel_connected;
if (detaching || !connected) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: detaching %s, channel connected %s",
fakeif->iff_name,
(detaching ? "true" : "false"),
(connected ? "true" : "false"));
goto done;
}
ifp = fakeif->iff_ifp;
peer_ifp = fakeif->iff_peer;
if (peer_ifp != NULL) {
peer_fakeif = ifnet_get_if_fake(peer_ifp);
if (peer_fakeif != NULL) {
detaching = feth_is_detaching(peer_fakeif);
connected = peer_fakeif->iff_channel_connected;
if (detaching || !connected) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"peer %s: detaching %s, "
"channel connected %s",
peer_fakeif->iff_name,
(detaching ? "true" : "false"),
(connected ? "true" : "false"));
goto done;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"peer_fakeif is NULL");
goto done;
}
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "peer_ifp is NULL");
goto done;
}
*packetCount = *byteCount = 0;
kern_packet_t sph = *ph;
while (sph != 0) {
(*packetCount)++;
*byteCount += kern_packet_get_data_length(sph);
sph = kern_packet_get_next(sph);
}
feth_nx_tx_queue_deliver_pkt_chain(fakeif, *ph, nifs,
peer_fakeif, 0, 0);
*ph = 0;
done:
feth_unlock();
return 0;
}
static void
fill_qset_info_and_params(if_fake_ref fakeif, fake_llink *llink_info,
uint32_t qset_idx, struct kern_nexus_netif_llink_qset_init *qset_init,
bool is_def, bool is_low_latency)
{
fake_qset *qset_info = &llink_info->fl_qset[qset_idx];
qset_init->nlqi_flags =
(is_def ? KERN_NEXUS_NET_LLINK_QSET_DEFAULT : 0) |
(is_low_latency ? KERN_NEXUS_NET_LLINK_QSET_LOW_LATENCY : 0) |
KERN_NEXUS_NET_LLINK_QSET_AQM;
if (feth_in_wmm_mode(fakeif)) {
qset_init->nlqi_flags |= KERN_NEXUS_NET_LLINK_QSET_WMM_MODE;
qset_init->nlqi_num_txqs = IFF_NUM_TX_QUEUES_WMM_MODE;
qset_init->nlqi_num_rxqs = IFF_NUM_RX_QUEUES_WMM_MODE;
} else {
qset_init->nlqi_num_txqs = 1;
qset_init->nlqi_num_rxqs = 1;
}
qset_info->fqs_tx_queue_cnt = qset_init->nlqi_num_txqs;
qset_info->fqs_rx_queue_cnt = qset_init->nlqi_num_rxqs;
/* These are needed for locating the peer qset */
qset_info->fqs_llink_idx = llink_info->fl_idx;
qset_info->fqs_idx = qset_idx;
}
static void
fill_llink_info_and_params(if_fake_ref fakeif, uint32_t llink_idx,
struct kern_nexus_netif_llink_init *llink_init, uint32_t llink_id,
struct kern_nexus_netif_llink_qset_init *qset_init, uint32_t qset_cnt,
uint32_t flags)
{
fake_llink *llink_info = &fakeif->iff_llink[llink_idx];
uint32_t i;
bool create_ll_qset = if_fake_low_latency && (llink_idx != 0);
for (i = 0; i < qset_cnt; i++) {
fill_qset_info_and_params(fakeif, llink_info, i,
&qset_init[i], i == 0, create_ll_qset && i == 1);
}
llink_info->fl_idx = llink_idx;
/* This doesn't have to be the same as llink_idx */
llink_info->fl_id = llink_id;
llink_info->fl_qset_cnt = qset_cnt;
llink_init->nli_link_id = llink_id;
llink_init->nli_num_qsets = qset_cnt;
llink_init->nli_qsets = qset_init;
llink_init->nli_flags = flags;
llink_init->nli_ctx = llink_info;
}
static errno_t
create_non_default_llinks(if_fake_ref fakeif)
{
struct kern_nexus *nx;
fake_nx_t fnx = &fakeif->iff_nx;
struct kern_nexus_netif_llink_init llink_init;
struct kern_nexus_netif_llink_qset_init qset_init[FETH_MAX_QSETS];
errno_t err;
uint64_t llink_id;
uint32_t i;
nx = nx_find(fnx->fnx_instance, FALSE);
if (nx == NULL) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s: nx not found", fakeif->iff_name);
return ENXIO;
}
/* Default llink starts at index 0 */
for (i = 1; i < if_fake_llink_cnt; i++) {
llink_id = (uint64_t)i;
/*
* The llink_init and qset_init structures are reused for
* each llink creation.
*/
fill_llink_info_and_params(fakeif, i, &llink_init,
llink_id, qset_init, if_fake_qset_cnt, 0);
err = kern_nexus_netif_llink_add(nx, &llink_init);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s: llink add failed, error %d",
fakeif->iff_name, err);
goto fail;
}
fakeif->iff_llink_cnt++;
}
nx_release(nx);
return 0;
fail:
for (i = 0; i < fakeif->iff_llink_cnt; i++) {
int error;
fake_llink * __single ll = &fakeif->iff_llink[i];
error = kern_nexus_netif_llink_remove(nx, ll->fl_id);
if (error != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
"%s: llink remove failed, llink_id 0x%llx, "
"error %d", fakeif->iff_name,
ll->fl_id, error);
}
ll->fl_id = 0;
}
fakeif->iff_llink_cnt = 0;
nx_release(nx);
return err;
}
static errno_t
create_netif_llink_provider_and_instance(if_fake_ref fakeif,
struct ifnet_init_eparams * init_params, ifnet_t *ifp,
uuid_t * provider, uuid_t * instance)
{
errno_t err;
nexus_controller_t controller = kern_nexus_shared_controller();
struct kern_nexus_net_init net_init;
struct kern_nexus_netif_llink_init llink_init;
struct kern_nexus_netif_llink_qset_init qsets[FETH_MAX_QSETS];
nexus_name_t provider_name;
nexus_attr_t nexus_attr = NULL;
struct kern_nexus_netif_provider_init prov_init = {
.nxnpi_version = KERN_NEXUS_DOMAIN_PROVIDER_NETIF,
.nxnpi_flags = NXPIF_VIRTUAL_DEVICE,
.nxnpi_pre_connect = feth_nx_pre_connect,
.nxnpi_connected = feth_nx_connected,
.nxnpi_pre_disconnect = feth_nx_pre_disconnect,
.nxnpi_disconnected = feth_nx_disconnected,
.nxnpi_qset_init = feth_nx_qset_init,
.nxnpi_qset_fini = feth_nx_qset_fini,
.nxnpi_queue_init = feth_nx_queue_init,
.nxnpi_queue_fini = feth_nx_queue_fini,
.nxnpi_tx_qset_notify = feth_nx_tx_qset_notify,
.nxnpi_config_capab = feth_nx_capab_config,
.nxnpi_queue_tx_push = feth_nx_queue_tx_push
};
err = kern_nexus_attr_create(&nexus_attr);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"nexus attribute creation failed, error %d", err);
goto failed;
}
err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_ANONYMOUS, 1);
VERIFY(err == 0);
snprintf((char *)provider_name, sizeof(provider_name),
"com.apple.netif.%s", fakeif->iff_name);
err = kern_nexus_controller_register_provider(controller,
feth_nx_dom_prov,
provider_name,
(struct kern_nexus_provider_init *)&prov_init,
sizeof(prov_init),
nexus_attr,
provider);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"register provider failed, error %d", err);
goto failed;
}
bzero(&net_init, sizeof(net_init));
net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
net_init.nxneti_flags = 0;
net_init.nxneti_eparams = init_params;
net_init.nxneti_lladdr = NULL;
net_init.nxneti_prepare = feth_netif_prepare;
net_init.nxneti_rx_pbufpool = fakeif->iff_rx_pp;
net_init.nxneti_tx_pbufpool = fakeif->iff_tx_pp;
/*
* Assume llink id is same as the index for if_fake.
* This is not required for other drivers.
*/
_CASSERT(NETIF_LLINK_ID_DEFAULT == 0);
fill_llink_info_and_params(fakeif, 0, &llink_init,
NETIF_LLINK_ID_DEFAULT, qsets, if_fake_qset_cnt,
KERN_NEXUS_NET_LLINK_DEFAULT);
net_init.nxneti_llink = &llink_init;
err = kern_nexus_controller_alloc_net_provider_instance(controller,
*provider, fakeif, NULL, instance, &net_init, ifp);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"alloc_net_provider_instance failed, %d", err);
kern_nexus_controller_deregister_provider(controller,
*provider);
uuid_clear(*provider);
goto failed;
}
fakeif->iff_llink_cnt++;
if (if_fake_llink_cnt > 1) {
err = create_non_default_llinks(fakeif);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"create_non_default_llinks failed, %d", err);
feth_detach_netif_nexus(fakeif);
goto failed;
}
}
if (feth_supports_tso(fakeif)) {
if ((err = feth_set_tso_offload(*ifp)) != 0) {
goto failed;
}
}
failed:
if (nexus_attr != NULL) {
kern_nexus_attr_destroy(nexus_attr);
}
return err;
}
static errno_t
feth_attach_netif_nexus(if_fake_ref fakeif,
struct ifnet_init_eparams * init_params, ifnet_t *ifp)
{
errno_t error;
fake_nx_t nx = &fakeif->iff_nx;
error = feth_packet_pool_make(fakeif);
if (error != 0) {
return error;
}
if (if_fake_llink_cnt == 0) {
return create_netif_provider_and_instance(fakeif, init_params,
ifp, &nx->fnx_provider, &nx->fnx_instance);
} else {
return create_netif_llink_provider_and_instance(fakeif,
init_params, ifp, &nx->fnx_provider,
&nx->fnx_instance);
}
}
static void
remove_non_default_llinks(if_fake_ref fakeif)
{
struct kern_nexus *nx;
fake_nx_t fnx = &fakeif->iff_nx;
uint32_t i;
if (fakeif->iff_llink_cnt <= 1) {
return;
}
nx = nx_find(fnx->fnx_instance, FALSE);
if (nx == NULL) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"%s: nx not found", fakeif->iff_name);
return;
}
/* Default llink (at index 0) is freed separately */
for (i = 1; i < fakeif->iff_llink_cnt; i++) {
int err;
err = kern_nexus_netif_llink_remove(nx, fakeif->
iff_llink[i].fl_id);
if (err != 0) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"%s: llink remove failed, llink_id 0x%llx, "
"error %d", fakeif->iff_name,
fakeif->iff_llink[i].fl_id, err);
}
fakeif->iff_llink[i].fl_id = 0;
}
fakeif->iff_llink_cnt = 0;
nx_release(nx);
}
static void
detach_provider_and_instance(uuid_t provider, uuid_t instance)
{
nexus_controller_t controller = kern_nexus_shared_controller();
errno_t err;
if (!uuid_is_null(instance)) {
err = kern_nexus_controller_free_provider_instance(controller,
instance);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"free_provider_instance failed %d", err);
}
uuid_clear(instance);
}
if (!uuid_is_null(provider)) {
err = kern_nexus_controller_deregister_provider(controller,
provider);
if (err != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"deregister_provider %d", err);
}
uuid_clear(provider);
}
return;
}
static void
feth_detach_netif_nexus(if_fake_ref fakeif)
{
fake_nx_t fnx = &fakeif->iff_nx;
remove_non_default_llinks(fakeif);
detach_provider_and_instance(fnx->fnx_provider, fnx->fnx_instance);
}
#endif /* SKYWALK */
/**
** feth interface routines
**/
static void
feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp)
{
errno_t error;
ifnet_offload_t offload = 0;
ifnet_set_addrlen(ifp, ETHER_ADDR_LEN);
ifnet_set_baudrate(ifp, 0);
ifnet_set_mtu(ifp, ETHERMTU);
ifnet_set_flags(ifp,
IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX,
0xffff);
ifnet_set_hdrlen(ifp, sizeof(struct ether_header));
if ((fakeif->iff_flags & IFF_FLAGS_LRO) != 0) {
offload |= IFNET_LRO;
}
if ((fakeif->iff_flags & IFF_FLAGS_HWCSUM) != 0) {
offload |= IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP |
IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6;
}
if (feth_supports_tso(fakeif)) {
offload |= IFNET_TSO_IPV4 | IFNET_TSO_IPV6;
}
if (feth_supports_vlan_tagging(fakeif)) {
offload |= IFNET_VLAN_TAGGING;
} else if (feth_supports_vlan_mtu(fakeif)) {
offload |= IFNET_VLAN_MTU;
}
error = ifnet_set_offload(ifp, offload);
if (error != 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"ifnet_set_offload(%s, 0x%x) failed, %d",
ifp->if_xname, offload, error);
} else {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"ifnet_set_offload(%s, 0x%x) succeeded",
ifp->if_xname, offload);
}
}
static void
interface_link_event(ifnet_t ifp, u_int32_t event_code)
{
struct event {
u_int32_t ifnet_family;
u_int32_t unit;
char if_name[IFNAMSIZ];
};
_Alignas(struct kern_event_msg) char message[sizeof(struct kern_event_msg) + sizeof(struct event)] = { 0 };
struct kern_event_msg *header = (struct kern_event_msg*)message;
struct event *data = (struct event *)(header + 1);
header->total_size = sizeof(message);
header->vendor_code = KEV_VENDOR_APPLE;
header->kev_class = KEV_NETWORK_CLASS;
header->kev_subclass = KEV_DL_SUBCLASS;
header->event_code = event_code;
data->ifnet_family = ifnet_family(ifp);
data->unit = (u_int32_t)ifnet_unit(ifp);
strlcpy(data->if_name, ifnet_name(ifp), IFNAMSIZ);
ifnet_event(ifp, header);
}
static if_fake_ref
ifnet_get_if_fake(ifnet_t ifp)
{
return (if_fake_ref)ifnet_softc(ifp);
}
static int
feth_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params)
{
bool bsd_mode;
int error;
if_fake_ref fakeif;
struct ifnet_init_eparams feth_init;
fake_llink *iff_llink;
ifnet_t ifp;
uint8_t mac_address[ETHER_ADDR_LEN];
bool multi_buflet;
iff_pktpool_mode_t pktpool_mode;
bool tso_support;
/* make local copy of globals needed to make consistency checks below */
bsd_mode = (if_fake_bsd_mode != 0);
multi_buflet = (if_fake_multibuflet != 0);
tso_support = (if_fake_tso_support != 0);
pktpool_mode = if_fake_pktpool_mode;
if (!bsd_mode) {
/* consistency checks */
if (if_fake_llink_cnt == 0 &&
strbufcmp(sk_ll_prefix, FAKE_ETHER_NAME) == 0) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
"feth used as ifname prefix but logical link "
"support in feth is disabled.");
return EINVAL;
}
if (tso_support && pktpool_mode != IFF_PP_MODE_GLOBAL) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"TSO mode requires global packet pool mode");
return EINVAL;
}
if (multi_buflet && pktpool_mode == IFF_PP_MODE_PRIVATE_SPLIT) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
"multi-buflet not supported for split rx & tx pool");
return EINVAL;
}
}
iff_llink = kalloc_type(fake_llink, FETH_MAX_LLINKS, Z_WAITOK_ZERO);
if (iff_llink == NULL) {
return ENOBUFS;
}
fakeif = kalloc_type(struct if_fake, Z_WAITOK_ZERO_NOFAIL);
fakeif->iff_llink = iff_llink;
fakeif->iff_retain_count = 1;
#define FAKE_ETHER_NAME_LEN (sizeof(FAKE_ETHER_NAME) - 1)
_CASSERT(FAKE_ETHER_NAME_LEN == 4);
bcopy(FAKE_ETHER_NAME, mac_address, FAKE_ETHER_NAME_LEN);
mac_address[ETHER_ADDR_LEN - 2] = (unit & 0xff00) >> 8;
mac_address[ETHER_ADDR_LEN - 1] = unit & 0xff;
if (bsd_mode) {
fakeif->iff_flags |= IFF_FLAGS_BSD_MODE;
}
if (if_fake_hwcsum != 0) {
fakeif->iff_flags |= IFF_FLAGS_HWCSUM;
}
if (if_fake_lro != 0) {
fakeif->iff_flags |= IFF_FLAGS_LRO;
}
if (if_fake_vlan_tagging != 0) {
/* support VLAN tagging in hardware */
feth_set_supports_vlan_tagging(fakeif);
} else {
/* support VLAN mtu-sized packets */
feth_set_supports_vlan_mtu(fakeif);
}
if (if_fake_separate_frame_header != 0) {
fakeif->iff_flags |= IFF_FLAGS_SEPARATE_FRAME_HEADER;
}
fakeif->iff_max_mtu = get_max_mtu(bsd_mode, if_fake_max_mtu);
fakeif->iff_fcs = if_fake_fcs;
fakeif->iff_trailer_length = if_fake_trailer_length;
/* use the interface name as the unique id for ifp recycle */
if ((unsigned int)
snprintf(fakeif->iff_name, sizeof(fakeif->iff_name), "%s%d",
ifc->ifc_name, unit) >= sizeof(fakeif->iff_name)) {
feth_release(fakeif);
return EINVAL;
}
bzero(&feth_init, sizeof(feth_init));
feth_init.ver = IFNET_INIT_CURRENT_VERSION;
feth_init.len = sizeof(feth_init);
if (feth_in_bsd_mode(fakeif)) {
if (if_fake_txstart != 0) {
feth_init.start = feth_start;
} else {
feth_init.flags |= IFNET_INIT_LEGACY;
feth_init.output = feth_output;
}
if (tso_support) {
feth_set_supports_tso(fakeif);
}
}
#if SKYWALK
else {
feth_init.flags |= IFNET_INIT_SKYWALK_NATIVE;
/*
* Currently we support WMM mode only for Skywalk native
* interface.
*/
if (if_fake_wmm_mode != 0) {
fakeif->iff_flags |= IFF_FLAGS_WMM_MODE;
}
if (multi_buflet) {
fakeif->iff_flags |= IFF_FLAGS_MULTIBUFLETS;
}
fakeif->iff_pp_mode = pktpool_mode;
if (tso_support) {
feth_set_supports_tso(fakeif);
}
fakeif->iff_tx_headroom = if_fake_tx_headroom;
fakeif->iff_adv_interval = if_fake_if_adv_interval;
if (fakeif->iff_adv_interval > 0) {
feth_init.flags |= IFNET_INIT_IF_ADV;
}
fakeif->iff_tx_drop_rate = if_fake_tx_drops;
fakeif->iff_tx_completion_mode = if_tx_completion_mode;
fakeif->iff_tx_exp_policy = if_fake_tx_exp_policy;
}
feth_init.tx_headroom = fakeif->iff_tx_headroom;
#endif /* SKYWALK */
if (if_fake_nxattach == 0) {
feth_init.flags |= IFNET_INIT_NX_NOAUTO;
}
feth_init.uniqueid = fakeif->iff_name;
feth_init.uniqueid_len = strlen(fakeif->iff_name);
feth_init.name = ifc->ifc_name;
feth_init.unit = unit;
feth_init.family = IFNET_FAMILY_ETHERNET;
feth_init.type = IFT_ETHER;
feth_init.demux = ether_demux;
feth_init.add_proto = ether_add_proto;
feth_init.del_proto = ether_del_proto;
feth_init.check_multi = ether_check_multi;
feth_init.framer_extended = ether_frameout_extended;
feth_init.softc = fakeif;
feth_init.ioctl = feth_ioctl;
feth_init.set_bpf_tap = NULL;
feth_init.detach = feth_if_free;
feth_init.broadcast_addr = etherbroadcastaddr;
feth_init.broadcast_len = ETHER_ADDR_LEN;
if (feth_in_bsd_mode(fakeif)) {
error = ifnet_allocate_extended(&feth_init, &ifp);
if (error) {
feth_release(fakeif);
return error;
}
feth_ifnet_set_attrs(fakeif, ifp);
if (feth_supports_tso(fakeif)) {
feth_set_tso_mtu(ifp, IP_MAXPACKET, IP_MAXPACKET);
}
}
#if SKYWALK
else {
if (feth_in_wmm_mode(fakeif)) {
feth_init.output_sched_model =
IFNET_SCHED_MODEL_DRIVER_MANAGED;
}
error = feth_attach_netif_nexus(fakeif, &feth_init, &ifp);
if (error != 0) {
feth_release(fakeif);
return error;
}
/* take an additional reference to ensure that it doesn't go away */
feth_retain(fakeif);
fakeif->iff_ifp = ifp;
}
#endif /* SKYWALK */
fakeif->iff_media_count = MIN(default_media_words_count, IF_FAKE_MEDIA_LIST_MAX);
bcopy(default_media_words, fakeif->iff_media_list,
fakeif->iff_media_count * sizeof(fakeif->iff_media_list[0]));
if (feth_in_bsd_mode(fakeif)) {
error = ifnet_attach(ifp, NULL);
if (error) {
ifnet_release(ifp);
feth_release(fakeif);
return error;
}
fakeif->iff_ifp = ifp;
}
ifnet_set_lladdr(ifp, mac_address, sizeof(mac_address));
/* attach as ethernet */
bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
return 0;
}
static int
feth_clone_destroy(ifnet_t ifp)
{
if_fake_ref fakeif;
#if SKYWALK
boolean_t nx_attached = FALSE;
#endif /* SKYWALK */
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL || feth_is_detaching(fakeif)) {
feth_unlock();
return 0;
}
feth_set_detaching(fakeif);
#if SKYWALK
nx_attached = !feth_in_bsd_mode(fakeif);
#endif /* SKYWALK */
feth_unlock();
#if SKYWALK
if (nx_attached) {
feth_detach_netif_nexus(fakeif);
feth_release(fakeif);
}
#endif /* SKYWALK */
feth_config(ifp, NULL);
ifnet_detach(ifp);
return 0;
}
static void
feth_enqueue_input(ifnet_t ifp, struct mbuf * m)
{
struct ifnet_stat_increment_param stats = {};
stats.packets_in = 1;
stats.bytes_in = (uint32_t)mbuf_pkthdr_len(m) + ETHER_HDR_LEN;
ifnet_input(ifp, m, &stats);
}
static int
feth_add_mbuf_trailer(struct mbuf *m, void *trailer, size_t trailer_len)
{
int ret;
ASSERT(trailer_len <= FETH_TRAILER_LENGTH_MAX);
ret = m_append(m, trailer_len, (caddr_t)trailer);
if (ret == 1) {
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%zuB trailer added", trailer_len);
return 0;
}
FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT, "m_append failed");
return ENOTSUP;
}
static int
feth_add_mbuf_fcs(struct mbuf *m)
{
uint32_t pkt_len, offset = 0;
uint32_t crc = 0;
int err = 0;
ASSERT(sizeof(crc) == ETHER_CRC_LEN);
pkt_len = m->m_pkthdr.len;
struct mbuf *iter = m;
while (iter != NULL && offset < pkt_len) {
uint32_t frag_len = iter->m_len;
ASSERT(frag_len <= (pkt_len - offset));
crc = crc32(crc, mtod(iter, void *), frag_len);
offset += frag_len;
iter = iter->m_next;
}
err = feth_add_mbuf_trailer(m, &crc, ETHER_CRC_LEN);
if (err != 0) {
return err;
}
m->m_flags |= M_HASFCS;
return 0;
}
static void
feth_output_common(ifnet_t ifp, struct mbuf * m, ifnet_t peer,
iff_flags_t flags, bool fcs, void *trailer, size_t trailer_len)
{
void * frame_header;
if ((flags & IFF_FLAGS_HWCSUM) != 0) {
m->m_pkthdr.csum_data = 0xffff;
m->m_pkthdr.csum_flags =
CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
CSUM_IP_CHECKED | CSUM_IP_VALID;
}
(void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0);
if (trailer_len != 0 && trailer != NULL) {
feth_add_mbuf_trailer(m, trailer, trailer_len);
}
if (fcs) {
feth_add_mbuf_fcs(m);
}
if ((flags & IFF_FLAGS_SEPARATE_FRAME_HEADER) != 0) {
m = m_copyup(m, ETHER_HDR_LEN, 0);
if (m == NULL) {
FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT, "m_copyup failed");
goto done;
}
frame_header = mbuf_data(m);
mbuf_pkthdr_setheader(m, frame_header);
m_adj(m, ETHER_HDR_LEN);
FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
"%s: frame 0x%llx data 0x%llx len %ld",
ifp->if_xname,
(uint64_t)VM_KERNEL_ADDRPERM(frame_header),
(uint64_t)VM_KERNEL_ADDRPERM(mtod(m, void *)),
mbuf_len(m));
} else {
frame_header = mbuf_data(m);
mbuf_pkthdr_setheader(m, frame_header);
_mbuf_adjust_pkthdr_and_data(m, ETHER_HDR_LEN);
}
/* tap it out */
if (ifp->if_bpf != NULL) {
fake_bpf_tap_out(ifp, m, frame_header);
}
/* tap it in */
if (peer->if_bpf != NULL) {
fake_bpf_tap_in(peer, m, frame_header);
}
(void)mbuf_pkthdr_setrcvif(m, peer);
feth_enqueue_input(peer, m);
done:
return;
}
static void
feth_start(ifnet_t ifp)
{
if_fake_ref fakeif;
iff_flags_t flags = 0;
bool fcs;
struct mbuf * m;
ifnet_t peer = NULL;
size_t trailer_len;
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
feth_unlock();
return;
}
if (fakeif->iff_start_busy) {
feth_unlock();
return;
}
peer = fakeif->iff_peer;
flags = fakeif->iff_flags;
fcs = fakeif->iff_fcs;
trailer_len = fakeif->iff_trailer_length;
fakeif->iff_start_busy = TRUE;
feth_unlock();
for (;;) {
if (ifnet_dequeue(ifp, &m) != 0) {
break;
}
if (peer == NULL) {
m_freem(m);
continue;
}
if (m != NULL) {
feth_output_common(ifp, m, peer, flags, fcs,
feth_trailer, trailer_len);
}
}
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif != NULL) {
fakeif->iff_start_busy = FALSE;
}
feth_unlock();
}
static int
feth_output(ifnet_t ifp, struct mbuf * m)
{
if_fake_ref fakeif;
iff_flags_t flags;
bool fcs;
size_t trailer_len;
ifnet_t peer = NULL;
if (m == NULL) {
return 0;
}
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif != NULL) {
peer = fakeif->iff_peer;
flags = fakeif->iff_flags;
fcs = fakeif->iff_fcs;
trailer_len = fakeif->iff_trailer_length;
}
feth_unlock();
if (peer == NULL) {
m_freem(m);
ifnet_stat_increment_out(ifp, 0, 0, 1);
return 0;
}
feth_output_common(ifp, m, peer, flags, fcs, feth_trailer, trailer_len);
return 0;
}
static int
feth_config(ifnet_t ifp, ifnet_t peer)
{
int connected = FALSE;
int disconnected = FALSE;
int error = 0;
if_fake_ref fakeif = NULL;
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
error = EINVAL;
goto done;
}
if (peer != NULL) {
/* connect to peer */
if_fake_ref peer_fakeif;
peer_fakeif = ifnet_get_if_fake(peer);
if (peer_fakeif == NULL) {
error = EINVAL;
goto done;
}
if (feth_is_detaching(fakeif) ||
feth_is_detaching(peer_fakeif) ||
peer_fakeif->iff_peer != NULL ||
fakeif->iff_peer != NULL) {
error = EBUSY;
goto done;
}
#if SKYWALK
if (fakeif->iff_pp_mode !=
peer_fakeif->iff_pp_mode) {
error = EINVAL;
goto done;
}
#endif /* SKYWALK */
fakeif->iff_peer = peer;
peer_fakeif->iff_peer = ifp;
connected = TRUE;
} else if (fakeif->iff_peer != NULL) {
/* disconnect from peer */
if_fake_ref peer_fakeif;
peer = fakeif->iff_peer;
peer_fakeif = ifnet_get_if_fake(peer);
if (peer_fakeif == NULL) {
/* should not happen */
error = EINVAL;
goto done;
}
fakeif->iff_peer = NULL;
peer_fakeif->iff_peer = NULL;
disconnected = TRUE;
}
done:
feth_unlock();
/* generate link status event if we connect or disconnect */
if (connected) {
interface_link_event(ifp, KEV_DL_LINK_ON);
interface_link_event(peer, KEV_DL_LINK_ON);
} else if (disconnected) {
interface_link_event(ifp, KEV_DL_LINK_OFF);
interface_link_event(peer, KEV_DL_LINK_OFF);
}
return error;
}
static int
feth_set_media(ifnet_t ifp, struct if_fake_request * iffr)
{
if_fake_ref fakeif;
int error;
if (iffr->iffr_media.iffm_count > IF_FAKE_MEDIA_LIST_MAX) {
/* list is too long */
return EINVAL;
}
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
error = EINVAL;
goto done;
}
fakeif->iff_media_count = iffr->iffr_media.iffm_count;
bcopy(iffr->iffr_media.iffm_list, fakeif->iff_media_list,
iffr->iffr_media.iffm_count * sizeof(fakeif->iff_media_list[0]));
#if 0
/* XXX: "auto-negotiate" active with peer? */
/* generate link status event? */
fakeif->iff_media_current = iffr->iffr_media.iffm_current;
#endif
error = 0;
done:
feth_unlock();
return error;
}
static int
if_fake_request_copyin(user_addr_t user_addr,
struct if_fake_request *iffr, u_int32_t len)
{
int error;
if (user_addr == USER_ADDR_NULL || len < sizeof(*iffr)) {
error = EINVAL;
goto done;
}
error = copyin(user_addr, iffr, sizeof(*iffr));
if (error != 0) {
goto done;
}
if (iffr->iffr_reserved[0] != 0 || iffr->iffr_reserved[1] != 0 ||
iffr->iffr_reserved[2] != 0 || iffr->iffr_reserved[3] != 0) {
error = EINVAL;
goto done;
}
done:
return error;
}
static int
feth_set_drvspec(ifnet_t ifp, uint32_t cmd, u_int32_t len,
user_addr_t user_addr)
{
int error;
struct if_fake_request iffr;
ifnet_t peer;
switch (cmd) {
case IF_FAKE_S_CMD_SET_PEER:
error = if_fake_request_copyin(user_addr, &iffr, len);
if (error != 0) {
break;
}
if (iffr.iffr_peer_name[0] == '\0') {
error = feth_config(ifp, NULL);
break;
}
/* ensure nul termination */
iffr.iffr_peer_name[IFNAMSIZ - 1] = '\0';
peer = ifunit(iffr.iffr_peer_name);
if (peer == NULL) {
error = ENXIO;
break;
}
if (ifnet_type(peer) != IFT_ETHER) {
error = EINVAL;
break;
}
if (strcmp(ifnet_name(peer), FAKE_ETHER_NAME) != 0) {
error = EINVAL;
break;
}
error = feth_config(ifp, peer);
break;
case IF_FAKE_S_CMD_SET_MEDIA:
error = if_fake_request_copyin(user_addr, &iffr, len);
if (error != 0) {
break;
}
error = feth_set_media(ifp, &iffr);
break;
case IF_FAKE_S_CMD_SET_DEQUEUE_STALL:
error = if_fake_request_copyin(user_addr, &iffr, len);
if (error != 0) {
break;
}
error = feth_enable_dequeue_stall(ifp,
iffr.iffr_dequeue_stall);
break;
default:
error = EOPNOTSUPP;
break;
}
return error;
}
static int
feth_get_drvspec(ifnet_t ifp, u_int32_t cmd, u_int32_t len,
user_addr_t user_addr)
{
int error = EOPNOTSUPP;
if_fake_ref fakeif;
struct if_fake_request iffr;
ifnet_t peer;
switch (cmd) {
case IF_FAKE_G_CMD_GET_PEER:
if (len < sizeof(iffr)) {
error = EINVAL;
break;
}
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
feth_unlock();
error = EOPNOTSUPP;
break;
}
peer = fakeif->iff_peer;
feth_unlock();
bzero(&iffr, sizeof(iffr));
if (peer != NULL) {
strlcpy(iffr.iffr_peer_name,
if_name(peer),
sizeof(iffr.iffr_peer_name));
}
error = copyout(&iffr, user_addr, sizeof(iffr));
break;
default:
break;
}
return error;
}
union ifdrvu {
struct ifdrv32 *ifdrvu_32;
struct ifdrv64 *ifdrvu_64;
void *ifdrvu_p;
};
static int
feth_ioctl(ifnet_t ifp, u_long cmd, void * data)
{
unsigned int count;
struct ifdevmtu * devmtu_p;
union ifdrvu drv;
uint32_t drv_cmd;
uint32_t drv_len;
boolean_t drv_set_command = FALSE;
int error = 0;
struct ifmediareq32 * ifmr;
struct ifreq * ifr;
if_fake_ref fakeif;
int status;
user_addr_t user_addr;
ifr = (struct ifreq *)data;
switch (cmd) {
case SIOCSIFADDR:
ifnet_set_flags(ifp, IFF_UP, IFF_UP);
break;
case SIOCGIFMEDIA32:
case SIOCGIFMEDIA64:
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
feth_unlock();
return EOPNOTSUPP;
}
status = (fakeif->iff_peer != NULL)
? (IFM_AVALID | IFM_ACTIVE) : IFM_AVALID;
ifmr = (struct ifmediareq32 *)data;
user_addr = (cmd == SIOCGIFMEDIA64) ?
((struct ifmediareq64 *)data)->ifmu_ulist :
CAST_USER_ADDR_T(((struct ifmediareq32 *)data)->ifmu_ulist);
count = ifmr->ifm_count;
ifmr->ifm_active = (fakeif->iff_peer != NULL)
? FAKE_DEFAULT_MEDIA : IFM_ETHER;
ifmr->ifm_current = IFM_ETHER;
ifmr->ifm_mask = 0;
ifmr->ifm_status = status;
if (user_addr == USER_ADDR_NULL) {
ifmr->ifm_count = fakeif->iff_media_count;
} else if (count > 0) {
if (count > fakeif->iff_media_count) {
count = fakeif->iff_media_count;
}
ifmr->ifm_count = count;
error = copyout(&fakeif->iff_media_list, user_addr,
count * sizeof(int));
}
feth_unlock();
break;
case SIOCGIFDEVMTU:
devmtu_p = &ifr->ifr_devmtu;
devmtu_p->ifdm_current = ifnet_mtu(ifp);
devmtu_p->ifdm_max = feth_max_mtu(ifp);
devmtu_p->ifdm_min = IF_MINMTU;
break;
case SIOCSIFMTU:
if ((unsigned int)ifr->ifr_mtu > feth_max_mtu(ifp) ||
ifr->ifr_mtu < IF_MINMTU) {
error = EINVAL;
} else {
error = ifnet_set_mtu(ifp, ifr->ifr_mtu);
}
break;
case SIOCSDRVSPEC32:
case SIOCSDRVSPEC64:
error = proc_suser(current_proc());
if (error != 0) {
break;
}
drv_set_command = TRUE;
OS_FALLTHROUGH;
case SIOCGDRVSPEC32:
case SIOCGDRVSPEC64:
drv.ifdrvu_p = data;
if (cmd == SIOCGDRVSPEC32 || cmd == SIOCSDRVSPEC32) {
drv_cmd = drv.ifdrvu_32->ifd_cmd;
drv_len = drv.ifdrvu_32->ifd_len;
user_addr = CAST_USER_ADDR_T(drv.ifdrvu_32->ifd_data);
} else {
drv_cmd = drv.ifdrvu_64->ifd_cmd;
drv_len = drv.ifdrvu_64->ifd_len;
user_addr = drv.ifdrvu_64->ifd_data;
}
if (drv_set_command) {
error = feth_set_drvspec(ifp, drv_cmd, drv_len,
user_addr);
} else {
error = feth_get_drvspec(ifp, drv_cmd, drv_len,
user_addr);
}
break;
case SIOCSIFLLADDR:
error = ifnet_set_lladdr(ifp, ifr->ifr_addr.sa_data,
ifr->ifr_addr.sa_len);
break;
case SIOCSIFFLAGS:
if ((ifp->if_flags & IFF_UP) != 0) {
/* marked up, set running if not already set */
if ((ifp->if_flags & IFF_RUNNING) == 0) {
/* set running */
error = ifnet_set_flags(ifp, IFF_RUNNING,
IFF_RUNNING);
}
} else if ((ifp->if_flags & IFF_RUNNING) != 0) {
/* marked down, clear running */
error = ifnet_set_flags(ifp, 0, IFF_RUNNING);
}
break;
case SIOCADDMULTI:
case SIOCDELMULTI:
error = 0;
break;
case SIOCSIFCAP: {
uint32_t cap;
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL ||
(fakeif->iff_flags & IFF_FLAGS_LRO) == 0) {
feth_unlock();
return EOPNOTSUPP;
}
feth_unlock();
cap = (ifr->ifr_reqcap & IFCAP_LRO) != 0 ? IFCAP_LRO : 0;
error = ifnet_set_capabilities_enabled(ifp, cap, IFCAP_LRO);
break;
}
default:
error = EOPNOTSUPP;
break;
}
return error;
}
static void
feth_if_free(ifnet_t ifp)
{
if_fake_ref fakeif;
if (ifp == NULL) {
return;
}
feth_lock();
fakeif = ifnet_get_if_fake(ifp);
if (fakeif == NULL) {
feth_unlock();
return;
}
ifp->if_softc = NULL;
#if SKYWALK
VERIFY(fakeif->iff_doorbell_tcall == NULL);
#endif /* SKYWALK */
feth_unlock();
feth_release(fakeif);
ifnet_release(ifp);
return;
}
__private_extern__ void
if_fake_init(void)
{
int error;
#if SKYWALK
(void)feth_register_nexus_domain_provider();
#endif /* SKYWALK */
error = if_clone_attach(&feth_cloner);
if (error != 0) {
return;
}
return;
}