~emersion/mrsh

15b900b051d250e98ce09be325b10bf2c4b3f436 — Drew DeVault 2 years ago 24cd296
Refactor tests and introduce conformance tests

This establishes test/conformance/, which contains tests which
exhaustively test each feature for comformance with the POSIX
specification down to the last detail. These tests should be useful for
verifying other shells as well, and do not require a test harness or a
reference shell (we still use both, though).

I'll DRY this up a bit once we support functions.
8 files changed, 99 insertions(+), 23 deletions(-)

A test/conformance/README
A test/conformance/if.sh
D test/data/if.sh
R test/{test.sh => harness.sh}
R test/data/{loop.sh => sh}
M test/meson.build
R test/data/{subshell.sh => ell.sh}
R test/data/{word.sh => sh}
A test/conformance/README => test/conformance/README +3 -0
@@ 0,0 1,3 @@
Tests in this directory test each specified behavior of a feature in the POSIX
specification. If mrsh passes such a test, it is considered compliant with the
appropriate part of the spec.

A test/conformance/if.sh => test/conformance/if.sh +78 -0
@@ 0,0 1,78 @@
#!/bin/sh
# Reference stdout:
# pass
# pass
# pass
# pass
# pass
# pass

echo >&2 "-> if..fi with true condition"
if true
then
	echo pass
fi

echo >&2 "-> if..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
fi
echo pass

echo >&2 "-> if..else..fi with true condition"
if true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo >&2 "-> if..else..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
else
	echo pass
fi

echo >&2 "-> if..else..fi with true condition"
if true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo >&2 "-> if..else..elif..fi with false condition"
if false
then
	echo >&2 "fail: This branch should not have run" && exit 1
elif true
then
	echo pass
else
	echo >&2 "fail: This branch should not have run" && exit 1
fi

echo >&2 "-> test exit status"
if true
then
	( exit 10 )
else
	( exit 20 )
fi
[ $# -eq 10 ] || { echo >&2 "fail: Expected status code = 10" && exit 1; }
if false
then
	( exit 10 )
else
	( exit 20 )
fi
[ $# -eq 20 ] || { echo >&2 "fail: Expected status code = 20" && exit 1; }

echo >&2 "-> test alternate syntax"
# These tests are only expected to parse, they do not make assertions
if true; then true; fi
if true; then true; else true; fi
if true; then true; elif true; then true; else true; fi

D test/data/if.sh => test/data/if.sh +0 -10
@@ 1,10 0,0 @@
#!/bin/sh

if true; then echo 1a; fi
if true; then echo 2a; else echo 2b; fi
if false; then echo 3a; else echo 3b; fi
if false; then echo 4a; elif true; then echo 4b; else echo 4c; fi

if false; then
	echo 5a
fi

R test/test.sh => test/harness.sh +8 -8
@@ 1,16 1,16 @@
#!/bin/sh

dir=$(dirname "$0")
testcase="$dir/$1"

mrsh="$1"
ref="$2"
testcase="$dir/$3"

mrsh_out=$("$mrsh" "$testcase")
env
echo >&2 "Running with mrsh"
mrsh_out=$("$MRSH" "$testcase")
mrsh_ret=$?
ref_out=$("$ref" "$testcase")
echo >&2 "Running with reference shell"
ref_out=$("$REF_SH" "$testcase")
ref_ret=$?
if [ "$mrsh_ret" != "$ref_ret" ] || [ "$mrsh_out" != "$ref_out" ] ; then
if [ $mrsh_ret -ne $ref_ret ] || [ "$mrsh_out" != "$ref_out" ]
then
	echo >&2 "$testcase: mismatch"
	echo >&2 ""
	echo >&2 "mrsh: $mrsh_ret"

R test/data/loop.sh => test/loop.sh +0 -0

M test/meson.build => test/meson.build +10 -5
@@ 1,8 1,9 @@
test_exe = find_program('./test.sh')
ref_exe = find_program('sh', required: false)
harness = find_program('./harness.sh')
ref_sh = find_program('sh', required: false)

test_files = [
	'if.sh',
	'conformance/if.sh',

	'loop.sh',
	'subshell.sh',
	'word.sh',


@@ 11,7 12,11 @@ test_files = [
foreach test_file : test_files
	test(
		test_file,
		test_exe,
		args: [mrsh_exe.full_path(), ref_exe.path(), 'data/'+test_file],
		harness,
		env: [
			'MRSH=@0@'.format(mrsh_exe.full_path()),
			'REF_SH=@0@'.format(ref_sh.path()),
		],
		args: [test_file],
	)
endforeach

R test/data/subshell.sh => test/subshell.sh +0 -0

R test/data/word.sh => test/word.sh +0 -0