~brenns10/funlisp

funlisp/test.py -rw-r--r-- 2.6 KiB
8947d954Stephen Brennan Release v1.2.0 1 year, 3 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/usr/bin/env python3
"""
Run tests to verify correctness, memory safety, etc.
"""
import argparse
import difflib
import glob
import os
import re
import subprocess
import sys

ERROR_EXITCODE = 211


def get_expected_code_and_output(script):
    reading_output = False
    output = ''
    output_re = re.compile(r'; OUTPUT\((\d+)\)')
    with open(script, 'r') as f:
        for line in f:
            if reading_output:
                output += line[line.index('; ') + 2:]
            elif output_re.match(line):
                code = int(output_re.match(line).group(1))
                reading_output = True

    return code, output


def run_test_script(script, runner):
    command = [
        'valgrind',
        '-q',
        '--error-exitcode={}'.format(ERROR_EXITCODE),
        runner,
        script,
    ]
    proc = subprocess.Popen(
        command, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate()
    return proc.returncode, stdout, stderr


def test_case(script, runner):
    print('[{}] {}: '.format(runner, script), end='')
    sys.stdout.flush()

    exp_code, exp_stdout = get_expected_code_and_output(script)
    assert exp_code != ERROR_EXITCODE
    act_code, act_stdout, act_stderr = run_test_script(script, runner)
    act_stdout = act_stdout.decode('utf-8')
    act_stderr = act_stderr.decode('utf-8')

    if act_code == ERROR_EXITCODE:
        print('MEM ERROR')
        print('## STDOUT')
        print(act_stdout)
        print('## STDERR')
        print(act_stderr)
        return False
    elif act_code != exp_code:
        print('FAIL (returned {}, expected {})'.format(
            act_code, exp_code))
        print('## STDOUT')
        print(act_stdout)
        print('## STDERR')
        print(act_stderr)
        return False
    elif act_stdout.strip() != exp_stdout.strip():
        print('FAIL (incorrect output)')
        exp_lines = [l + '\n' for l in exp_stdout.split('\n')]
        act_lines = [l + '\n' for l in act_stdout.split('\n')]
        for line in difflib.unified_diff(exp_lines, act_lines):
            sys.stdout.write(line)
        return False
    else:
        print('PASS')
        return True


def run_tests(test_files, runner):
    for script in test_files:
        if not test_case(script, runner):
            sys.exit(1)


if __name__ == '__main__':
    ap = argparse.ArgumentParser('run tests for funlisp')
    ap.add_argument('directory', help='directory of tests to run')
    ap.add_argument('--runner', '-r', help='binary to run with',
        default='bin/funlisp')
    args = ap.parse_args()
    run_tests(
        glob.glob(os.path.join(args.directory, '*.lisp')),
        args.runner)