#!/usr/bin/python3
# format_vm_parameter_validation.py
# Pretty-print the output of tests/vm/vm_parameter_validation.c
#
# usage:
# vm_parameter_validation | format_vm_parameter_validation.py
import re
import sys
import copy
import itertools
# magic return values used for in-band signalling
# fixme duplicated in vm_parameter_validation.c
# fixme also duplicated in other_return_values below
RESULT_SUCCESS = 0
RESULT_BUSTED = -99
RESULT_IGNORED = -98
RESULT_ZEROSIZE = -97
RESULT_PANIC = -96
RESULT_GUARD = -95
RESULT_MISMATCH = -94
RESULT_OUT_PARAM_BAD = -93
RESULT_MACH_SEND_INVALID_MEMORY = 0x1000000c
# output formatting
format_result = {
RESULT_SUCCESS : ' .',
RESULT_BUSTED : ' **',
RESULT_MISMATCH : ' ##',
RESULT_IGNORED : ' ',
RESULT_ZEROSIZE : ' o',
RESULT_PANIC : ' pp',
RESULT_GUARD : ' gg',
RESULT_OUT_PARAM_BAD: ' ot',
RESULT_MACH_SEND_INVALID_MEMORY : ' mi',
}
format_default = '%3d'
format_col_width = 3
format_empty_col = format_col_width * ' '
format_indent_width = 4
format_indent = format_indent_width * ' '
# record the result of one trial:
# ret: the return value from the tested function
# parameters: array of the input parameter names for that trial
# (for example ["start PGSZ-2", "size -1"])
class Result:
def __init__(self, new_ret, new_parameters):
self.ret = new_ret
self.parameters = new_parameters
def __repr__(self):
return str(self.ret) + " = " + str(self.parameters)
# record the results of all trials in one test
# testname: the name of the test (including the function being tested)
# config: a string describing OS, CPU, etc
# compat: code for error compatibility
# results: an array of Result, one per trial
class Test:
def __init__(self, new_name, new_config, new_compat, new_results = []):
self.testname = new_name
self.config = new_config
self.compat = new_compat
self.results = new_results
# print column labels under some output
# example output given indent=2 col_width=4 labels=[foo,bar,baz,qux]:
# | | | |
# | | | qux
# | | baz
# | bar
# foo
def print_column_labels(labels, indent_width, col_width):
indent = indent_width * ' '
empty_column = '|' + (col_width-1) * ' '
unprinted = len(labels)
print(indent + unprinted*empty_column)
for label in reversed(labels):
unprinted -= 1
print(indent + unprinted*empty_column + label)
# pretty-print one function return code
def print_one_result(ret):
if ret in format_result:
print(format_result[ret], end='')
else:
print(format_default % (ret), end='')
# choose the appropriate error code table for a test
# (either errno_return_values or kern_return_values)
def error_code_values_for_test(test):
errno_fns = ['mprotect', 'msync', 'minherit', 'mincore', 'mlock', 'munlock',
'mmap', 'munmap', 'mremap_encrypted', 'vslock', 'vsunlock',
'madvise', 'useracc']
for fn in errno_fns:
if test.testname.startswith(fn):
return errno_return_values
else:
return kern_return_values
# print a helpful description of the return values seen in results
# fixme these won't include RESULT_MISMATCH
def print_legend(test):
# find all error codes represented in the results
codes = {}
for result in test.results:
codes[result.ret] = True
known_return_values = error_code_values_for_test(test)
# print the names of the detected error codes
output = []
for code in sorted(codes.keys()):
if code in known_return_values:
output.append(str(code) + ': ' + known_return_values[code])
elif code in other_return_values:
output.append(other_return_values[code])
elif code != 0:
output.append(str(code) + ': ????')
print(format_indent + '(' + ', '.join(output) + ')')
# display names for error codes returned in errno
errno_return_values = {
1: 'EPERM',
9: 'EBADF',
12: 'ENOMEM',
13: 'EACCES',
14: 'EFAULT',
22: 'EINVAL',
}
# display names for error codes returned in kern_return_t
kern_return_values = {
1: 'KERN_INVALID_ADDRESS',
2: 'KERN_PROTECTION_FAILURE',
3: 'KERN_NO_SPACE',
4: 'KERN_INVALID_ARGUMENT',
5: 'KERN_FAILURE',
6: 'KERN_RESOURCE_SHORTAGE',
7: 'KERN_NOT_RECEIVER',
8: 'KERN_NO_ACCESS',
9: 'KERN_MEMORY_FAILURE',
10: 'KERN_MEMORY_ERROR',
11: 'KERN_ALREADY_IN_SET',
12: 'KERN_NOT_IN_SET',
13: 'KERN_NAME_EXISTS',
14: 'KERN_ABORTED',
15: 'KERN_INVALID_NAME',
16: 'KERN_INVALID_TASK',
17: 'KERN_INVALID_RIGHT',
18: 'KERN_INVALID_VALUE',
19: 'KERN_UREFS_OVERFLOW',
20: 'KERN_INVALID_CAPABILITY',
21: 'KERN_RIGHT_EXISTS',
22: 'KERN_INVALID_HOST',
23: 'KERN_MEMORY_PRESENT',
24: 'KERN_MEMORY_DATA_MOVED',
25: 'KERN_MEMORY_RESTART_COPY',
26: 'KERN_INVALID_PROCESSOR_SET',
27: 'KERN_POLICY_LIMIT',
28: 'KERN_INVALID_POLICY',
29: 'KERN_INVALID_OBJECT',
30: 'KERN_ALREADY_WAITING',
31: 'KERN_DEFAULT_SET',
32: 'KERN_EXCEPTION_PROTECTED',
33: 'KERN_INVALID_LEDGER',
34: 'KERN_INVALID_MEMORY_CONTROL',
35: 'KERN_INVALID_SECURITY',
36: 'KERN_NOT_DEPRESSED',
37: 'KERN_TERMINATED',
38: 'KERN_LOCK_SET_DESTROYED',
39: 'KERN_LOCK_UNSTABLE',
40: 'KERN_LOCK_OWNED',
41: 'KERN_LOCK_OWNED_SELF',
42: 'KERN_SEMAPHORE_DESTROYED',
43: 'KERN_RPC_SERVER_TERMINATED',
44: 'KERN_RPC_TERMINATE_ORPHAN',
45: 'KERN_RPC_CONTINUE_ORPHAN',
46: 'KERN_NOT_SUPPORTED',
47: 'KERN_NODE_DOWN',
48: 'KERN_NOT_WAITING',
49: 'KERN_OPERATION_TIMED_OUT',
50: 'KERN_CODESIGN_ERROR',
51: 'KERN_POLICY_STATIC',
52: 'KERN_INSUFFICIENT_BUFFER_SIZE',
53: 'KERN_DENIED',
54: 'KERN_MISSING_KC',
55: 'KERN_INVALID_KC',
56: 'KERN_NOT_FOUND',
100: 'KERN_RETURN_MAX',
-304: 'MIG_BAD_ARGUMENTS (server type check failure)',
0x1000000c : 'MACH_SEND_INVALID_MEMORY',
}
# display names for the special return values used by the test machinery
other_return_values = {
RESULT_BUSTED: format_result[RESULT_BUSTED].lstrip() + ': trial broken, not performed',
RESULT_IGNORED: '<empty> trial ignored, not performed',
RESULT_ZEROSIZE: format_result[RESULT_ZEROSIZE].lstrip() + ': size == 0',
RESULT_PANIC: format_result[RESULT_PANIC].lstrip() + ': trial is believed to panic, not performed',
RESULT_GUARD: format_result[RESULT_GUARD].lstrip() + ': trial is believed to throw EXC_GUARD, not performed',
RESULT_OUT_PARAM_BAD: format_result[RESULT_OUT_PARAM_BAD].lstrip() + ': trial set incorrect values to out parameters',
}
# inside line, replace 'return 123' with 'return ERR_CODE_NAME'
def replace_error_code_return(test, line):
known_return_values = error_code_values_for_test(test)
for code, name in known_return_values.items():
line = line.replace('return ' + str(code) + ';', 'return ' + name + ';')
return line
def dimensions(results):
if len(results) == 0:
return 0
return len(results[0].parameters)
# given one k-dimensional results
# return a list of k counts that is the size of each dimension
def count_each_dimension(results):
if len(results) == 0:
return []
first = results[0].parameters
k = dimensions(results)
counts = []
step = 1
for dim in range(k-1, -1, -1):
count = round(len(results) / step)
for i in range(0, len(results), step):
cur = results[i].parameters
if i != 0 and cur[dim] == first[dim]:
count = round(i / step)
break;
step *= count
counts.append(count)
counts.reverse()
return counts;
# Reduce one k-dimensional results to many (k-1) dimensional results
# Yields a sequence of [results, name] pairs
# where results has k-1 dimensions
# and name is the parameter name from the removed dimension
def iterate_dimension(results, dim = 0):
if len(results) == 0:
return
k = dimensions(results)
dim_counts = count_each_dimension(results)
inner_count = 1
for d in range(dim+1, k):
inner_count *= dim_counts[d]
outer_step = len(results)
for d in range(0, dim):
outer_step = int(outer_step / dim_counts[d])
for r in range(dim_counts[dim]):
start = r * inner_count
name = results[start].parameters[dim]
new_results = []
for i in range(start, len(results), outer_step):
for j in range(inner_count):
new_result = copy.deepcopy(results[i+j])
del new_result.parameters[dim]
new_results.append(new_result)
yield [new_results, name]
# Print the results of a test that has two parameters (for example a test of start/size)
# If overrides!=None, use any non-SUCCESS return values from override in place of the other results.
def print_results_2D(results, overrides=None):
# complain if results and override have different dimensions
if overrides:
if len(overrides) != len(results):
print("WARNING: override results have a different height; overrides ignored")
for i, result in enumerate(results):
if len(overrides[i].parameters) != len(result.parameters):
print("WARNING: override results have a different width; overrides ignored")
columns = []
prev_row_label = ''
first_row_label = ''
for i, result in enumerate(results):
if overrides: override = overrides[i].ret
if first_row_label == '':
# record first row's name so we can use it to find columns
# (assumes every row has the same column labels)
first_row_label = result.parameters[0]
if result.parameters[0] == first_row_label:
# record column names in the first row
columns.append(result.parameters[1])
if result.parameters[0] != prev_row_label:
# new row
if prev_row_label != '': print(format_indent + prev_row_label)
print(format_indent, end='')
prev_row_label = result.parameters[0]
if overrides and override != RESULT_SUCCESS:
print_one_result(override)
else:
print_one_result(result.ret)
if prev_row_label: print(format_indent + prev_row_label)
print_column_labels(columns, format_indent_width + format_col_width - 1, format_col_width)
def print_results_2D_try_condensed(results):
if 0 == len(results):
return
singleton = results[0].ret
if any([result.ret != singleton for result in results]):
print_results_2D(results)
return
# will print as condensed
rows = set()
cols = set()
for result in results:
rows.add(result.parameters[0].split()[1])
cols.add(result.parameters[1].split()[1])
print_one_result(result.ret)
print(" for all pairs")
def print_results_3D(results, testname):
# foreach parameter[1], print 2D table of parameter[0] and parameter[2]
for results2D, name in iterate_dimension(results, 1):
print(testname + ': ' + name)
print_results_2D(results2D)
# foreach parameter[0], print 2D table of parameter[1] and parameter[2]
# This is redundant but can be useful for human readers.
for results2D, name in iterate_dimension(results, 0):
print(testname + ': ' + name)
print_results_2D(results2D)
def print_results_4D(results):
x, y, z = '', '', ''
# Make a map[{3rd_param, 4th_param, ...}] = {all options}
# For now, we print 2d tables of 1st, 2nd param for each possible combination of remaining values
map_of_results = {}
for _, result in enumerate(results):
k = tuple(result.parameters[2:])
if k not in map_of_results:
map_of_results[k] = [result]
else:
map_of_results[k].append(result)
# prepare to iterate
prev_matrix = []
iterable = []
for k, result_list in map_of_results.items():
one_2d_result = []
matrix = []
for result in result_list:
x = result.parameters[0]
y = result.parameters[1]
repl_result = Result(result.ret, (x, y))
one_2d_result.append(repl_result)
matrix.append(result.ret)
if matrix == prev_matrix:
# if the return codes are the same everywhere, we will print successive tables only once
# note that this assumes that the sets of 2D labels are the same everywhere, and doesn't check that assumption
iterable[-1][0].append(k)
else:
iterable.append(([k], one_2d_result))
prev_matrix = matrix
# print
for iter in iterable:
print(iter[0])
print_results_2D_try_condensed(iter[1])
# Print the results of a test that has two parameters
# (for example a test of addr only, or size only)
# If overrides!=None, use any non-SUCCESS return values from override in place of the other results.
def print_results_1D(results, overrides=None):
# complain if results and overrides have different dimensions
if overrides:
if len(overrides) != len(results):
print("WARNING: override results have a different height; overrides ignored")
for i, result in enumerate(results):
if len(overrides[i].parameters) != len(result.parameters):
print("WARNING: override results have a different width; overrides ignored")
for i, result in enumerate(results):
if overrides: override = overrides[i].ret
# indent, value, indent, label
print(format_indent, end='')
if overrides and override != RESULT_SUCCESS:
print_one_result(override)
else:
print_one_result(result.ret)
print(format_indent + result.parameters[0])
def print_results_nD(results, testname, overrides=None):
if (dimensions(results) == 1):
print_results_1D(results, overrides)
elif (dimensions(results) == 2):
print_results_2D(results, overrides)
elif dimensions(results) == 3:
print_results_3D(results, testname)
elif dimensions(results) == 4:
print_results_4D(results)
else:
print(format_indent + 'too many dimensions')
def main():
data = sys.stdin.readlines()
# remove any lines that don't start with "TESTNAME" or "TESTCONFIG" or "RESULT"
# (including darwintest output like "PASS" or "FAIL")
# and print them now
# Also verify that the counts of "TEST BEGIN" == "TEST END"
# (they will mismatch if a test suite crashed)
testbegincount = 0
testendcount = 0
testlines = []
for line in data:
unmodified_line = line
# count TEST BEGIN and TEST END
if ('TEST BEGIN' in line):
testbegincount += 1
if ('TEST END' in line):
testendcount += 1
# remove any T_LOG() timestamp prefixes and KTEST prefixes
line = re.sub('^\s*\d+:\d+:\d+ ', '', line)
line = re.sub('^\[KTEST\]\s+[A-Z]+\s+\d+\s+(\d+\s+)?\S+\s+\d+\s+', '', line)
line = line.lstrip()
if (line.startswith('TESTNAME') or line.startswith('RESULT')
or line.startswith('TESTCONFIG') or line.startswith('TESTCOMPAT')):
testlines.append(line) # line is test output
elif line == '':
pass # ignore empty lines
else:
print(unmodified_line, end='') # line is other output
# parse test output into Test and Result objects
testnum = 0
def group_by_test(line):
nonlocal testnum
if line.startswith('TESTNAME '):
testnum = testnum+1
return testnum
tests = []
for _, group in itertools.groupby(testlines, group_by_test):
lines = list(group)
name = lines.pop(0).removeprefix('TESTNAME ').rstrip()
config = lines.pop(0).removeprefix('TESTCONFIG ').rstrip()
compat = []
results = []
for line in lines:
if line.startswith('RESULT'):
components = line.removeprefix('RESULT ').rstrip().split(', ')
ret = int(components.pop(0))
results.append(Result(ret, components))
tests.append(Test(name, config, compat, results))
print('found %d tests' % (len(tests)))
# stats to print at the end
test_count = len(tests)
all_configurations = set()
# print test output
for test in tests:
# print test name and test config on separate lines
# `diff` handles this better than putting both on the same line
print('test ' + test.testname)
print(format_indent + 'config ' + test.config)
all_configurations.add(test.config)
if len(test.results) == 0:
print(format_indent + 'no results')
else:
print_legend(test)
print_results_nD(test.results, test.testname)
print('end ' + test.testname)
print()
print(str(test_count) + ' test(s) performed')
if (testbegincount != testendcount):
print('### error: %d TEST BEGINs, %d TEST ENDs - some tests may have crashed'
% (testbegincount, testendcount))
print(str(len(all_configurations)) + ' configuration(s) tested:')
for config in sorted(all_configurations):
print(format_indent + '[' + config + ']')
main()