/*
 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#ifndef _SKYWALK_OS_SKYWALK_PRIVATE_H
#define _SKYWALK_OS_SKYWALK_PRIVATE_H

#if defined(PRIVATE) || defined(XNU_KERNEL_PRIVATE)
/*
 * This file contains private interfaces for Skywalk, and should not
 * be included by code external to xnu kernel or libsystem_kernel.
 * The only exception to this is skywalk_cmds for the internal tools.
 */

/* branch prediction helpers */
#include <sys/cdefs.h>
#define SK_ALIGN64_CASSERT(type, field) \
	_Static_assert((__builtin_offsetof(type, field) % sizeof(uint64_t)) == 0, "incorrect alignment")

#if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE)
enum {
	SK_FEATURE_SKYWALK = 1ULL << 0,
	SK_FEATURE_DEVELOPMENT = 1ULL << 1,
	SK_FEATURE_DEBUG = 1ULL << 2,
	SK_FEATURE_NEXUS_FLOWSWITCH = 1ULL << 3,
	SK_FEATURE_NEXUS_UNUSED_4 = 1ULL << 4,
	SK_FEATURE_NEXUS_NETIF = 1ULL << 5,
	SK_FEATURE_NEXUS_USER_PIPE = 1ULL << 6,
	SK_FEATURE_NEXUS_KERNEL_PIPE = 1ULL << 7,
	SK_FEATURE_NEXUS_KERNEL_PIPE_LOOPBACK = 1ULL << 8,
	SK_FEATURE_DEV_OR_DEBUG = 1ULL << 9,
	SK_FEATURE_NETNS = 1ULL << 10,
	SK_FEATURE_PROTONS = 1ULL << 11,
};
#endif /* !KERNEL || BSD_KERNEL_PRIVATE */

/* valid flags for if_attach_nx */
#define IF_ATTACH_NX_NETIF_COMPAT               0x01    /* create compat netif */
#define IF_ATTACH_NX_FLOWSWITCH                 0x02    /* enable flowswitch */
#define IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT     0x04    /* enable fsw TCP/UDP netagent */
#define IF_ATTACH_NX_NETIF_NETAGENT             0x08    /* enable netif netagent */
#define IF_ATTACH_NX_NETIF_ALL                  0x10    /* don't restrict netif */
#define IF_ATTACH_NX_FSW_IP_NETAGENT            0x20    /* enable fsw IP netagent */

/*
 * Enabling Skywalk channel (user) networking stack fundamentally requires
 * the presence of netif nexus attached to the interface.  Native Skywalk
 * drivers will come with a netif nexus; non-native drivers will require us
 * to create a netif compat nexus, set through IF_ATTACH_NX_NETIF_COMPAT.
 *
 * The flowswitch nexus creation depends on the presence of netif nexus on
 * the interface.  Plumbing the flowswitch atop netif is required to connect
 * the interface to the host (kernel) networking stack; this is the default
 * behavior unless IF_ATTACH_NX_FLOWSWITCH is not set.
 *
 * To further allow for channel (user) networking stack, the netagent gets
 * enabled on the flowswitch by default, unless opted out by clearing the
 * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT flag. Note that
 * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT cannot be changed after boot, and so has
 * to be set by the if_attach_nx bootarg.
 */
#define SKYWALK_NETWORKING_ENABLED                              \
	(IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH |  \
	IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT | IF_ATTACH_NX_FSW_IP_NETAGENT)

/*
 * To partially enable Skywalk to only let the host (kernel) stack to work,
 * we set both IF_ATTACH_NX_NETIF_COMPAT and IF_ATTACH_NX_FLOWSWITCH flags,
 * leaving IF_ATTACH_NX_FSW_*_NETAGENT unset. This enables netif and
 * flowswitch flowswitch nexus on all eligible interfaces, except that channel
 * (user) networking stack will be disabled.
 */
#define SKYWALK_NETWORKING_BSD_ONLY     \
	(IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH)

/*
 * macOS default configuration for enabling support for interpose filters,
 * custom IP, custom ether type providers and user networking stack.
 */
#define SKYWALK_NETWORKING_MAC_OS     \
	(SKYWALK_NETWORKING_BSD_ONLY | IF_ATTACH_NX_FSW_IP_NETAGENT | \
	IF_ATTACH_NX_NETIF_NETAGENT | IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT)

/*
 * Disabling Skywalk networking is done by removing IF_ATTACH_NX_NETIF_COMPAT
 * and IF_ATTACH_NX_FSW_*_NETAGENT flags, and leaving IF_ATTACH_NX_FLOWSWITCH
 * set. This disables the compat netif instances and netagent, while still
 * allowing flowswitch nexus creation for native Skywalk drivers, since
 * otherwise they cease to function due to missing interaction with the host
 * (kernel) stack.
 */
#define SKYWALK_NETWORKING_DISABLED     IF_ATTACH_NX_FLOWSWITCH

#if !XNU_TARGET_OS_OSX
#ifdef __LP64__
#define IF_ATTACH_NX_DEFAULT    SKYWALK_NETWORKING_ENABLED
#else /* !__LP64__ */
#define IF_ATTACH_NX_DEFAULT    SKYWALK_NETWORKING_DISABLED
#endif /* !__LP64__ */
#else /* XNU_TARGET_OS_OSX */
#define IF_ATTACH_NX_DEFAULT    SKYWALK_NETWORKING_MAC_OS
#endif /* XNU_TARGET_OS_OSX */

#define SK_VERBOSE_SYSCTL       "kern.skywalk.verbose"
/*
 * Verbose flags.
 *
 * When adding new ones, consider the existing scheme based on the areas;
 * try to "fill in the holes" first before extending the range.
 */
#define SK_VERB_FLAGS_TABLE(X)  \
	X(SK_VERB_DEFAULT,		0)      /* 0x0000000000000001 */ \
	X(SK_VERB_DUMP,			1)      /* 0x0000000000000002 */ \
	X(SK_VERB_LOCKS,		2)      /* 0x0000000000000004 */ \
	X(SK_VERB_REFCNT,		3)      /* 0x0000000000000008 */ \
	X(SK_VERB_MEM,			4)      /* 0x0000000000000010 */ \
	X(SK_VERB_MEM_ARENA,		5)      /* 0x0000000000000020 */ \
	X(SK_VERB_MEM_CACHE,		6)      /* 0x0000000000000040 */ \
	X(SK_VERB_MEM_REGION,		7)      /* 0x0000000000000080 */ \
	X(SK_VERB_EVENTS,		8)      /* 0x0000000000000100 */ \
	X(SK_VERB_SYNC,			9)      /* 0x0000000000000200 */ \
	X(SK_VERB_NOTIFY,		10)     /* 0x0000000000000400 */ \
	X(SK_VERB_INTR,			11)     /* 0x0000000000000800 */ \
	X(__SK_VERB_12,			12)     /* 0x0000000000001000 */ \
	X(SK_VERB_DEV,			13)     /* 0x0000000000002000 */ \
	X(SK_VERB_HOST,			14)     /* 0x0000000000004000 */ \
	X(SK_VERB_USER,			15)     /* 0x0000000000008000 */ \
	X(SK_VERB_RX,			16)     /* 0x0000000000010000 */ \
	X(SK_VERB_TX,			17)     /* 0x0000000000020000 */ \
	X(SK_VERB_LOOKUP,		18)     /* 0x0000000000040000 */ \
	X(SK_VERB_RING,			19)     /* 0x0000000000080000 */ \
	X(SK_VERB_NETIF,		20)     /* 0x0000000000100000 */ \
	X(SK_VERB_NETIF_MIT,		21)     /* 0x0000000000200000 */ \
	X(SK_VERB_IOSK,			22)     /* 0x0000000000400000 */ \
	X(SK_VERB_CHANNEL,		23)     /* 0x0000000000800000 */ \
	X(SK_VERB_AQM,			25)     /* 0x0000000002000000 */ \
	X(SK_VERB_FSW,			24)     /* 0x0000000001000000 */ \
	X(SK_VERB_FSW_DP,		26)     /* 0x0000000004000000 */ \
	X(SK_VERB_LLINK,		27)     /* 0x0000000008000000 */ \
	X(SK_VERB_FLOW,			28)     /* 0x0000000010000000 */ \
	X(SK_VERB_FLOW_CLASSIFY,	29)     /* 0x0000000020000000 */ \
	X(SK_VERB_FLOW_TRACK,		30)     /* 0x0000000040000000 */ \
	X(SK_VERB_FLOW_ADVISORY,	31)     /* 0x0000000080000000 */ \
	X(SK_VERB_FLOW_ROUTE,		32)     /* 0x0000000100000000 */ \
	X(SK_VERB_FPD,		        33)     /* 0x0000000200000000 */ \
	X(__SK_VERB_34__,		34)     /* 0x0000000400000000 */ \
	X(__SK_VERF_35__,		35)     /* 0x0000000800000000 */ \
	X(SK_VERB_USER_PIPE,		36)     /* 0x0000001000000000 */ \
	X(SK_VERB_NA,			37)     /* 0x0000002000000000 */ \
	X(SK_VERB_KERNEL_PIPE,		38)     /* 0x0000004000000000 */ \
	X(SK_VERB_NS_PROTO,		39)     /* 0x0000008000000000 */ \
	X(SK_VERB_NS_TCP,		40)     /* 0x0000010000000000 */ \
	X(SK_VERB_NS_UDP,		41)     /* 0x0000020000000000 */ \
	X(SK_VERB_NS_IPV4,		42)     /* 0x0000040000000000 */ \
	X(SK_VERB_NS_IPV6,		43)     /* 0x0000080000000000 */ \
	X(SK_VERB_COPY,			44)     /* 0x0000100000000000 */ \
	X(SK_VERB_COPY_MBUF,		45)     /* 0x0000200000000000 */ \
	X(SK_VERB_MOVE,			46)     /* 0x0000400000000000 */ \
	X(SK_VERB_MOVE_MBUF,		47)     /* 0x0000800000000000 */ \
	X(SK_VERB_IP_FRAG,		48)     /* 0x0001000000000000 */ \
	X(SK_VERB_ERROR_INJECT,		49)     /* 0x0002000000000000 */ \
	X(SK_VERB_QOS,			50)     /* 0x0004000000000000 */ \
	X(SK_VERB_NXPORT,		51)     /* 0x0008000000000000 */ \
	X(SK_VERB_FILTER,		52)     /* 0x0010000000000000 */ \
	X(SK_VERB_VP,			53)     /* 0x0020000000000000 */ \
	X(SK_VERB_NETIF_POLL,		54)     /* 0x0040000000000000 */ \
	X(SK_VERB_DROP,			55)     /* 0x0080000000000000 */ \
	X(__SK_VERB_56__,		56)     /* 0x0100000000000000 */ \
	X(__SK_VERB_57__,		57)     /* 0x0200000000000000 */ \
	X(__SK_VERB_58__,		58)     /* 0x0400000000000000 */ \
	X(__SK_VERB_59__,		59)     /* 0x0800000000000000 */ \
	X(__SK_VERB_60__,		60)     /* 0x1000000000000000 */ \
	X(__SK_VERB_61__,		61)     /* 0x2000000000000000 */ \
	X(SK_VERB_PRIV,			62)     /* 0x4000000000000000 */ \
	X(SK_VERB_ERROR,		63)     /* 0x8000000000000000 */

#define EXPAND_TO_STRING(name, bitshift) #name,
#define EXPAND_TO_ENUMERATION(name, bitshift) name = (1ULL << bitshift),

static const char *sk_verb_flags_string[] = {
	SK_VERB_FLAGS_TABLE(EXPAND_TO_STRING)
};

enum SK_VERB_FLAGS {
	SK_VERB_FLAGS_TABLE(EXPAND_TO_ENUMERATION)
};

#define SK_VERB_FLAGS_STRINGS_MAX       \
	(sizeof (sk_verb_flags_string) / sizeof (sk_verb_flags_string[0]))

#undef  EXPAND_TO_STRING
#undef  EXPAND_TO_ENUMERATION

#ifdef KERNEL
#include <stdint.h>
#include <sys/types.h>
#include <sys/time.h>
#include <mach/vm_types.h>
#include <mach/vm_param.h>
#include <kern/cpu_number.h>
#include <pexpert/pexpert.h>
#include <os/log.h>

#define SK_LOG 1

#if SK_LOG
extern uint64_t sk_verbose;
extern os_log_t sk_log_handle;

#define SK_KVA(p) (void *)VM_KERNEL_ADDRHIDE(p)
#define SK_LOG_VAR(x) x

#define _SK_LOG(_type, _flag, _fmt, ...) do { \
	if (__improbable(((_flag) == SK_VERB_DEFAULT) || \
	    ((_flag) == SK_VERB_ERROR) || \
	    ((sk_verbose & (_flag)) == (_flag)))) { \
	        os_log_with_type(sk_log_handle, \
	            (_flag) == SK_VERB_ERROR ? OS_LOG_TYPE_ERROR : _type, \
	            "SK[%u]: %-30s " _fmt "\n", cpu_number(), __FUNCTION__, \
	            ##__VA_ARGS__); \
	} \
} while (0)

/* error log (captured by default) */
#define SK_ERR(_fmt, ...)       _SK_LOG(OS_LOG_TYPE_ERROR, SK_VERB_ERROR, _fmt, ##__VA_ARGS__)

/* error log with proc info (captured by default) */
#define SK_PERR(_p, _fmt, ...) do { \
	SK_ERR("%s(%d): " _fmt, sk_proc_name(_p), sk_proc_pid(_p), ##__VA_ARGS__); \
} while (0)

/* default log (captured by default) */
#define SK_D(_fmt, ...)         _SK_LOG(OS_LOG_TYPE_DEFAULT, SK_VERB_DEFAULT, _fmt, ##__VA_ARGS__)

/* debug log (enabled sk_verbose flag) */
#define SK_DF(_flag, _fmt, ...)  _SK_LOG(OS_LOG_TYPE_DEFAULT, (uint64_t)_flag, _fmt, ##__VA_ARGS__)

/* SK_DF with proc info */
#define SK_PDF(_flag, _p, _fmt, ...) do { \
	SK_DF(_flag, "%s(%d): " _fmt, sk_proc_name(_p), sk_proc_pid(_p), ##__VA_ARGS__); \
} while (0)

/* rate limited, lps indicates how many per second */
#define _SK_RD(_flag, _lps, _fmt, ...) do {                             \
	static int __t0, __now, __cnt;                                  \
	__now = (int)net_uptime();                                       \
	if (__t0 != __now) {                                            \
	        __t0 = __now;                                           \
	        __cnt = 0;                                              \
	}                                                               \
	if (__cnt++ < (_lps))                                           \
	        SK_DF(_flag, _fmt, ##__VA_ARGS__);                      \
} while (0)

#define SK_RDF(_flag, _lps, _fmt, ...)  \
	_SK_RD(_flag, _lps, _fmt, ##__VA_ARGS__)
#define SK_RD(_lps, _fmt, ...)          \
	SK_RDF(SK_VERB_DEFAULT, _lps, _fmt, ##__VA_ARGS__)
#define SK_RDERR(_lps, _fmt, ...)       \
	SK_RDF(SK_VERB_ERROR, _lps, _fmt, ##__VA_ARGS__)

/*
 * The compiler doesn't know that snprintf() supports %b format
 * specifier, so use our own wrapper to vsnprintf() here instead.
 */
#define sk_snprintf(str, size, format, ...)  ({ \
	_Pragma("clang diagnostic push")                                   \
	_Pragma("clang diagnostic ignored \"-Wformat-invalid-specifier\"") \
	_Pragma("clang diagnostic ignored \"-Wformat-extra-args\"")        \
	_Pragma("clang diagnostic ignored \"-Wformat\"")                   \
	snprintf(str, size, format, ## __VA_ARGS__)                        \
	_Pragma("clang diagnostic pop");                                   \
})

#else /* !SK_LOG */
#define SK_LOG_VAR(x)
#define SK_DF(_flag, _fmt, ...)         do { ((void)0); } while (0)
#define SK_D(_fmt, ...)                 do { ((void)0); } while (0)
#define SK_ERR(_fmt, ...)               do { ((void)0); } while (0)
#define SK_DSC(_p, _fmt, ...)           do { ((void)0); } while (0)
#define SK_RDF(_flag, _lps, _fmt, ...)  do { ((void)0); } while (0)
#define SK_RD(_lps, _fmt, ...)          do { ((void)0); } while (0)
#define SK_RDERR(_lps, _fmt, ...)       do { ((void)0); } while (0)
#endif /* ! SK_LOG */

#define SK_INLINE_ATTRIBUTE     __attribute__((always_inline))
#define SK_NO_INLINE_ATTRIBUTE  __attribute__((noinline))
#define SK_LOG_ATTRIBUTE        __attribute__((noinline, cold, not_tail_called))


#ifdef BSD_KERNEL_PRIVATE
#include <skywalk/core/skywalk_var.h>
#include <skywalk/lib/cuckoo_hashtable.h>
#include <skywalk/mem/skmem_var.h>
#include <skywalk/channel/os_channel_event.h>
#include <skywalk/channel/channel_var.h>
#include <skywalk/nexus/nexus_var.h>
#include <skywalk/packet/pbufpool_var.h>
#include <skywalk/packet/packet_var.h>
#endif /* BSD_KERNEL_PRIVATE */
#endif /* KERNEL */
#if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE)
#include <skywalk/skywalk_common.h>
#include <skywalk/os_nexus_private.h>
#include <skywalk/os_channel_private.h>
#include <skywalk/os_packet_private.h>
#include <skywalk/os_stats_private.h>
#endif /* !KERNEL || BSD_KERNEL_PRIVATE */
#endif /* PRIVATE || XNU_KERNEL_PRIVATE */
#endif /* _SKYWALK_OS_SKYWALK_PRIVATE_H */
