~emersion/mrsh

845da7c44465fda142ca324c6ff57613a8c39ea2 — Drew DeVault 6 months ago 96e8a90
test: overhaul conformance tests

This introduces comprehensive conformance testing for section 2.2 of the
POSIX shell standard. However, note that mrsh does not currently pass
these new tests. For what it's worth, neither does dash.
M meson_options.txt => meson_options.txt +7 -0
@@ 26,3 26,10 @@ option(
	value: 'sh',
	description: 'Reference shell used in tests',
)

option(
	'test-undefined-behavior',
	type: 'boolean',
	value: true,
	description: 'Run tests that assert undefined behavior fails',
)

A test/conformance/2.2-quoted-characters.sh => test/conformance/2.2-quoted-characters.sh +36 -0
@@ 0,0 1,36 @@
# 2.2.1 Escape Character (Backslash)
# The following must be quoted:
printf '%s\n' \|\&\;\<\>\(\)\$\`\\\"\'
# The following must be quoted only under certain conditions:
printf '%s\n' *?[#˜=%
# Escaping whitespace:
printf 'arg one: %s; arg two: %s\n' with\ space with\	tab
printf 'arg one: %s; arg two: %s\n' without \
	newline
printf 'arg one: %s; arg two: %s\n' without\
whitespace 'arg two'

# 2.2.2 Single quotes
printf '%s\n' '|&;<>()$`\"'

# 2.2.3 Double quotes
printf '%s\n' "|&;<>()'\""

# Parameter & arithmetic expansion should work:
lval=2
rval=3
printf '%s\n' "$lval + ${rval} = $((lval+rval))"
# Command expansion should work
printf '%s\n' "cmd 1: $(echo "cmd 1")"
printf '%s\n' "cmd 2: `echo "cmd 2"`"
# Command expansion should work, recursively
printf '%s\n' "cmd 3: $(echo $(echo "cmd 3"))"

# Backquotes should work
printf '%s\n' "cmd 4: `echo "cmd 4"`"

# Backslashes should escape the following:
printf '%s\n' "\$\`\"\\\
test"
# But not the following:
printf '%s\n' "\|\&\;\<\>\(\)\'"

A test/conformance/2.2-quoted-characters.stdout => test/conformance/2.2-quoted-characters.stdout +14 -0
@@ 0,0 1,14 @@
|&;<>()$`\"'
*?[#˜=%
arg one: with space; arg two: with	tab
arg one: without; arg two: newline
arg one: withoutwhitespace; arg two: arg two
|&;<>()$`\"
|&;<>()'"
2 + 3 = 5
cmd 1: cmd 1
cmd 2: cmd 2
cmd 3: cmd 3
cmd 4: cmd 4
$`"\test
\|\&\;\<\>\(\)\'

A test/conformance/2.2.2-nested-single-quotes.fail.sh => test/conformance/2.2.2-nested-single-quotes.fail.sh +3 -0
@@ 0,0 1,3 @@
# 2.2.2 Single quotes
# Single quotes cannot contain single quotes
printf '%s\n' '''

A test/conformance/2.2.3-alias-expansion.fail.sh => test/conformance/2.2.3-alias-expansion.fail.sh +4 -0
@@ 0,0 1,4 @@
# 2.2.3 Double quotes
# Should NOT apply alias expansion rules when finding the )
alias myalias="echo )"
var="$(myalias arg-two)"

A test/conformance/2.2.3-backquote-nonterminated-dquote.undefined.sh => test/conformance/2.2.3-backquote-nonterminated-dquote.undefined.sh +1 -0
@@ 0,0 1,1 @@
printf '%s\n' "cmd: `echo "`"

A test/conformance/2.2.3-backquote-nonterminated-squote.undefined.sh => test/conformance/2.2.3-backquote-nonterminated-squote.undefined.sh +1 -0
@@ 0,0 1,1 @@
printf '%s\n' "cmd: `echo '`"

A test/conformance/2.2.3-dquote-nonterminated-backquote.undefined.sh => test/conformance/2.2.3-dquote-nonterminated-backquote.undefined.sh +1 -0
@@ 0,0 1,1 @@
printf '%s\n' "`echo"

A test/conformance/harness.sh => test/conformance/harness.sh +21 -0
@@ 0,0 1,21 @@
#!/bin/sh
dir=$(dirname "$0")
testcase="$1"
stdout="${testcase%%.sh}.stdout"

# Set TEST_SHELL to quickly compare the conformance of different shells
mrsh=${TEST_SHELL:-$MRSH}

actual_out=$("$mrsh" "$testcase")
actual_ret=$?

if [ -f "$stdout" ]
then
	stdout="$(cat "$stdout")"
	if [ "$stdout" != "$actual_out" ]
	then
		exit 1
	fi
fi

exit $actual_ret

D test/conformance/if.sh => test/conformance/if.sh +0 -78
@@ 1,78 0,0 @@
#!/bin/sh
# Reference stdout:
# pass
# pass
# pass
# pass
# pass
# pass

echo "-> if..fi with true condition"
if true
then
	echo pass
fi

echo "-> if..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
fi
echo pass

echo "-> if..else..fi with true condition"
if true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo "-> if..else..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
else
	echo pass
fi

echo "-> if..else..fi with true condition"
if true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo "-> if..else..elif..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
elif true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo "-> test exit status"
if true
then
	( exit 10 )
else
	( exit 20 )
fi
[ $# -eq 10 ] || { echo >&2 "fail: Expected status code = 10" && exit 1; }
if false
then
	( exit 10 )
else
	( exit 20 )
fi
[ $# -eq 20 ] || { echo >&2 "fail: Expected status code = 20" && exit 1; }

echo "-> test alternate syntax"
# These tests are only expected to parse, they do not make assertions
if true; then true; fi
if true; then true; else true; fi
if true; then true; elif true; then true; else true; fi

A test/conformance/meson.build => test/conformance/meson.build +52 -0
@@ 0,0 1,52 @@
harness = find_program('./harness.sh')

test_files = [
	'2.2-quoted-characters.sh',
]

failures = [
	'2.2.2-nested-single-quotes.fail.sh',
	# TODO: https://github.com/emersion/mrsh/issues/145
	#'2.2.3-alias-expansion.fail.sh',
]

undefined = [
	'2.2.3-backquote-nonterminated-squote.undefined.sh',
	'2.2.3-backquote-nonterminated-dquote.undefined.sh',
	'2.2.3-dquote-nonterminated-backquote.undefined.sh',
]

testenv = [
	'MRSH=@0@'.format(mrsh_exe.full_path()),
]

foreach test_file : test_files
	test(
		'conformance/' + test_file,
		harness,
		env: testenv,
		args: [join_paths(meson.current_source_dir(), test_file)],
	)
endforeach

foreach test_file : failures
	test(
		'conformance/' + test_file,
		harness,
		env: testenv,
		args: [join_paths(meson.current_source_dir(), test_file)],
		should_fail: true,
	)
endforeach

if get_option('test-undefined-behavior')
	foreach test_file : undefined
		test(
			'conformance/' + test_file,
			harness,
			env: testenv,
			args: [join_paths(meson.current_source_dir(), test_file)],
			should_fail: true,
		)
	endforeach
endif

M test/meson.build => test/meson.build +2 -2
@@ 2,8 2,6 @@ harness = find_program('./harness.sh')
ref_sh = find_program(get_option('reference-shell'), required: false)

test_files = [
	'conformance/if.sh',

	'args.sh',
	'arithm.sh',
	'async.sh',


@@ 34,3 32,5 @@ foreach test_file : test_files
		args: [join_paths(meson.current_source_dir(), test_file)],
	)
endforeach

subdir('conformance')