~nch/gamelog

ref: 75432213d3972f26c03ef636da68ed482f0ac908 gamelog/testy.lua -rwxr-xr-x 20.6 KiB View raw
75432213 — nc refactoring debug code out a bit 5 months ago
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
#!/usr/bin/env lua

-- **Testy** is a quick-and-dirty unit testing script for Lua modules
-- that tries to be as unobtrusive as possible. It loads the specified
-- modules and collects test functions from local variables by means
-- of debug hooks. Finally, those test functions are run to collect
-- and print statistics about passed/failed test assertions.
--
-- Nice features about this approach are:
-- *   By storing the test code side-by-side with your regular module
--     code it should be easier to keep those two in sync.
-- *   You can test internal/local functions without messing up your
--     public interface (because the test functions themselves are
--     local functions embedded in the module code as well).
-- *   If you don't load the module via the `testy.lua` script, the
--     local test functions and all test data just goes out of scope
--     and gets garbage-collected very quickly.
--
-- The current implementation consists of a single pure Lua file
-- compatible with Lua 5.1 and up, with no external dependencies.
--
-- The `testy.lua` [source code][1] is available on GitHub, and is
-- released under the [MIT license][2]. You can view [a nice HTML
-- version][3] of this file rendered by [Docco][4] on the GitHub
-- pages.
--
-- Test functions are identified by a `"test_"` prefix and use the
-- standard `assert` function or the new `testy_assert` function for
-- individual test assertions. Both functions just log failure/success
-- and print a visual indicator to the console, but they do *not*
-- terminate the program (of course `assert` still does when used
-- outside of a test function for compatibility).
--
-- Here is an example:
--
--     -- module1.lua
--     local M = {}
--
--     function M.func1()
--       return 1
--     end
--     
--     -- this is a test function for the module function `M.func1()`
--     local function test_func1()
--       assert( M.func1() == 1, "func1() should always return 1" )
--       assert( M.func1() ~= 2, "func1() should never return 2" )
--       assert( type( M.func1() ) == "number" )
--     end
--
--     function M.func2()
--       return 2
--     end
--     
--     -- this is a test function for the module function `M.func2()`
--     local function test_func2()
--       assert( M.func2() == 2 )
--       assert( M.func2() ~= M.func1() )
--     end
--     
--     return M
--
-- Using the `testy.lua` script on this file will get you the
-- following output:
--
--     $ testy.lua module1.lua
--     func1 ('module1.lua')   ...
--     func2 ('module1.lua')   ..
--     5 tests (5 ok, 0 failed, 0 errors)
--
-- **Testy** is a very minimal unit testing framework that lacks lots
-- of features that other unit testing frameworks have, but in return
-- you can start unit testing without a learning curve.
--
--   [1]: http://github.com/siffiejoe/lua-testy
--   [2]: http://opensource.org/licenses/MIT
--   [3]: http://siffiejoe.github.io/lua-testy/
--   [4]: http://jashkenas.github.io/docco/
--
-- ## Implementation
--

-- There are some obviously arbitrary design choices (like e.g. the
-- prefix of the test functions) that one might want to customize.
-- Those variables allow you to do just that:
local prefix = "test_" -- the prefix of test functions to look for
local pass_char, fail_char = ".", "X" -- output for passed/failed tests
local max_line = 72 -- where to wrap test output in the terminal
local gap = " " -- space between caption and first pass/fail_char
local fh = io.stderr -- file handle to print test output to

-- There's also some data that the `testy.lua` script needs to keep
-- track of, like module files, test functions, test failures, etc.:
local files, chunks, do_recursive, do_tap = {}, {}, false, false
local tests, test_functions = {}, {}
local n_tests, n_passed, n_errors = 0, 0, 0
local cursor_pos = 0
local locals = {}
local thischunk = debug.getinfo( 1, "f" ).func
local assert = assert -- we monkey-patch assert, so save the real one
-- try load some useful functions for actually writing the tests:
local extra_ok, extra = pcall( require, "testy.extra" )
if not extra_ok then
  extra = {}
end


-- During `assert` or `testy_assert` the test statistics are updated
-- and a visual indicator is printed to the console.
local function evaluate_test_assertion( finfo, cinfo, ok, ... )
  n_tests = n_tests + 1
  if do_tap then
    fh:write( ok and "" or "not ", "ok ", n_tests )
    local src, line = finfo.source, cinfo.currentline
    -- The test description is just the file and line number. In
    -- principle the `assert` message could be used here, but it
    -- often describes the error instead of the test case, so this
    -- could be weird.
    fh:write( " ", src, ":", line )
    -- However, if the assertion message starts with "# TODO" or
    -- "# SKIP", those directives are passed through to the TAP
    -- consumer.
    if type( (...) ) == "string" and
       ((...):match( "^#[\t ]*[Tt][Oo][Dd][Oo]" ) or
        (...):match( "^#[\t ]*[Ss][Kk][Ii][Pp]" )) then
      fh:write( " ", (...) )
    end
    -- In case the test failed, an additional diagnostic message is
    -- printed:
    if not ok then
      local msg = (...) ~= nil and tostring( (...) )
                               or "test assertion failed!"
      fh:write( "\n# Failed test (", src, " at line ", line, ": '",
                msg:gsub( "\n", "\n#\t" ), "')" )
    end
    fh:write( "\n" )
  else
    fh:write( ok and pass_char or fail_char )
    -- For nicer output the visual test indicators are wrapped at a
    -- certain line length (`max_line`).
    cursor_pos = (cursor_pos + 1) % max_line
    if cursor_pos == 0 then
      fh:write( "\n" )
    end
  end
  fh:flush()
  if ok then
    n_passed = n_passed + 1
    return ok, ...
  else
    -- Details of test failures are stored per test function and
    -- printed when all `assert`s in this test function are complete.
    -- This looks nicer on screen. (Another option would be to print
    -- all failure details at the very end.) For the TAP output the
    -- failure details are written out just after the failed test.
    local fail = {
      no = n_tests,
      line = cinfo.currentline,
      reason = (...) ~= nil and tostring( (...) ) or nil
    }
    finfo[ #finfo+1 ] = fail
  end
end


-- **Testy** provides a monkey-patched `assert` function that can be
-- used in test functions without killing the program on an assertion
-- failure. For compatibility, any call of this function outside of
-- test functions just uses the [original `assert` function][5] from
-- Lua's standard library. Usually this is exactly what you want, but
-- there may be certain situations where you want to move an `assert`
-- call to an extra function and still update test statistics (like
-- e.g. assertions in callbacks, or helper functions for assertions).
-- For these cases **Testy** also provides the new global function
-- `testy_assert`.
--
--   [5]: http://www.lua.org/manual/5.1/manual.html#pdf-assert
local function _G_assert( ok, ... )
  -- The `assert` replacement checks the call stack via the `debug`
  -- API to find the calling test function and some extra information
  -- for the test statistics.
  local info = debug.getinfo( 2, "fl" )
  local finfo = test_functions[ info.func or false ]
  if finfo then
    return evaluate_test_assertion( finfo, info, ok, ... )
  else
    return assert( ok, ... )
  end
end


-- `testy_assert` works similar to the `assert` replacement function,
-- but since calls to this function in non-test code are not an issue
-- (it is a new function), `testy_assert` works anywhere and can
-- always be used instead of plain `assert`. In certain situations it
-- *has* to be used to run **Testy** assertions, e.g.:
--
--     local function assert_equal( x, y )
--       testy_assert( x == y )  -- call in helper assertion function
--     end
--
--     local function test_mytest()
--       local function callback( x )
--         testy_assert( x == 1 ) -- call in callback
--       end
--       M.foreachi( { 1, 1, 1 }, callback )
--       assert_equal( 1, 1 )
--     end
--
-- Although the new `testy_assert` function is more general than the
-- monkey-patched `assert` function the latter is still made available
-- because:
--
-- *   Every Lua programmer can see what's going on, and it looks more
--     familiar.
-- *   Converting ad-hoc test code is easier.
-- *   Most test code can be run without using the `testy.lua` program
--     simply by adding a call to one or more test functions in the
--     module code.
-- *   Also `assert` is shorter than `testy_assert`. ;-)
local function _G_testy_assert( ok, ... )
  -- A `testy_assert` call also inspects the call stack to find the
  -- test function it belongs to, but since the restriction that it
  -- has to be called *directly* from the test function could be
  -- lifted, the entire call stack is searched from top to bottom.
  local info, i, finfo = debug.getinfo( 2, "fl" ), 3
  while info do
    if info.func == thischunk then break end
    finfo = test_functions[ info.func or false ]
    if finfo then break end
    info, i = debug.getinfo( i, "fl" ), i+1
  end
  if finfo then
    return evaluate_test_assertion( finfo, info, ok, ... )
  else
    error( "call to 'testy_assert' function outside of tests", 2 )
  end
end


-- The local test functions are collected via debug hooks from main
-- chunks only. This function checks that a debug hook belongs to
-- a main chunk.
local function main_chunk( lvl )
  lvl = lvl+1 -- skip stack level of this function
  local info, i = debug.getinfo( lvl, "Sf" ), lvl+2
  if not info or info.what ~= "main" or info.func == thischunk then
    return false
  end
  -- If the `-r` flag is in effect, any main chunk may contain test
  -- functions that will be collected. If `-r` is *not* in effect,
  -- only the main chunk executed directly by the `testy.lua` script
  -- will be scanned.
  if not do_recursive then
    info = debug.getinfo( lvl+1, "Sf" )
    while info and info.func ~= thischunk do
      if info.what == "main" then
        return false
      end
      info, i = debug.getinfo( i, "Sf" ), i+1
    end
  end
  return true
end


-- Usually a return hook would be the perfect place to collect
-- information about local variables because all variables have been
-- defined and contain their final values. Unfortunately all current
-- PUC-Rio Lua versions (5.1.5, 5.2.4, and 5.3.0) clobber the local
-- variables before the return hook is executed. As a consequence,
-- **Testy** saves the current state of the local variables on every
-- line using an additional line hook, and uses that saved information
-- in the return hook to identify test functions. Sadly that can be
-- very inefficient, especially if the code executes a lot of lines
-- (e.g. using a loop), but top level module code normally doesn't do
-- that (it usually contains mostly function definitions). The test
-- functions themselves are executed without debug hooks and thus run
-- at full speed, so if you need to run a lot of code to prepare your
-- test cases, better move that code into the first test function.
local function line_ret_hook( event, no )
  if event ~= "tail_return" and main_chunk( 2 ) then
    local info = debug.getinfo( 2, "Sf" )
    if event == "line" then
      local locs = {}
      local i, name, value = 2, debug.getlocal( 2, 1 )
      while name do
        if #name >= #prefix and
           type( value ) == "function" and
           name:sub( 1, #prefix ) == prefix then
          locs[ #locs+1 ] = {
            caption = name:sub( #prefix+1 ):gsub( "_+", function( u )
              return #u == 1 and " " or u:sub( 2 )
            end ),
            name = name,
            func = value,
            source = info.short_src,
          }
        end
        i, name, value = i+1, debug.getlocal( 2, i )
      end
      locals[ info.func ] = locs
    else -- return hook
      for _,tdata in ipairs( locals[ info.func ] or {} ) do
        tests[ #tests+1 ] = tdata
        test_functions[ tdata.func ] = tdata
      end
    end
  end
end


-- When using the line hook to collect local variables, under some
-- circumstances the last local isn't picked up when the definition
-- is the last statement in the chunk. To circumvent that problem
-- this function first tries to load the code with an extra `return`
-- statement appended. Only if that fails (which it will if the code
-- already contains a final `return`), the original code is loaded.
-- Obviously this approach will fail when loading binary chunks, so
-- this is currently unsupported in **Testy** (although it will work
-- in most cases).
local function loadfile_with_extra_return( fname )
  local f, msg = io.open( fname, "rb" )
  if not f then
    return nil, msg
  end
  local s = f:read( "*a" )
  if not s then
    return nil, "input/ouput error"
  end
  -- `loadstring`/`load` won't handle shebang lines like `loadfile`
  -- does, so the shebang line has to be removed.
  s = s:gsub( "^#[^\n]*", "") .. "\nreturn\n"
  local c, msg = (loadstring or load)( s, "@"..fname )
  if c then
    return c
  else
    return loadfile( fname )
  end
end


-- The enhanced/modified Lua searcher below needs the [standard Lua
-- function `package.searchpath`][6] available in Lua 5.2+ to locate
-- Lua files. For Lua 5.1 a backport is provided:
--
--   [6]: http://www.lua.org/manual/5.2/manual.html#pdf-package.searchpath
local searchpath = package.searchpath
if not searchpath then
  local delim = package.config:match( "^(.-)\n" ):gsub( "%%", "%%%%" )

  function searchpath( name, path )
    local pname = name:gsub( "%.", delim ):gsub( "%%", "%%%%" )
    local msg = {}
    for subpath in path:gmatch( "[^;]+" ) do
      local fpath = subpath:gsub( "%?", pname )
      local f = io.open( fpath, "r" )
      if f then
        f:close()
        return fpath
      end
      msg[ #msg+1 ] = "\n\tno file '"..fpath.."'"
    end
    return nil, table.concat( msg )
  end
end


-- The issue about the missing last local definition in chunks also
-- applies to modules in case there is no explicit `return` statement
-- (which could be for a module using the deprecated `module` function
-- or a reimplementation thereof). The following replacement function
-- of the [standard Lua module searcher][7] uses the above mentioned
-- `loadfile_with_extra_return` to fix that.
--
--   [7]: http://www.lua.org/manual/5.2/manual.html#pdf-package.searchers
local function lua_searcher( modname )
  assert( type( modname ) == "string" )
  local fn, msg = searchpath( modname, package.path )
  if not fn then
    return msg
  end
  local mod, msg = loadfile_with_extra_return( fn )
  if not mod then
    error( "error loading module '"..modname.."' from file '"..fn..
           "':\n\t"..msg, 0 )
  end
  return mod, fn
end


-- The command line of `testy.lua` is inspected to collect command
-- line flags (currently only `-r` and `-t`) and all module/test files
-- that should be tested.
for i,a in ipairs( _G.arg ) do
  -- The `-r` command line flag causes **Testy** to collect the local
  -- test functions not only from the loaded files directly, but also
  -- recursively from `require`d modules.
  --
  -- The `-t` command line flag causes **Testy** to write out
  -- [TAP](http://testanything.org/tap-specification.html)-formatted
  -- output to the standard output stream. This way you can use
  -- other reporting tools like e.g. `prove`:
  --
  --     prove --exec "testy.lua -t" module1.lua
  if a == "-r" then
    do_recursive = true
  elseif a == "-t" then
    do_tap = true
    fh = io.stdout
  else
    files[ #files+1 ] = a
  end
  -- The arguments intended for the `testy.lua` script are removed
  -- from the `arg` table in case one of the loaded files also tries
  -- to process command line arguments.
  _G.arg[ i ] = nil
end

-- All collected module/test files are loaded and checked for syntax
-- errors. Errors at this stage are considered fatal and thus
-- terminate the test session.
for i,f in ipairs( files ) do
  chunks[ i ] = assert( loadfile_with_extra_return( f ) )
end

-- If the `-r` command line flag is in effect, the fix to `loadfile`
-- needs to be applied to `require`d modules as well. This is done by
-- replacing the standard Lua searcher function with the fixed
-- version from above.
if do_recursive then
  local searchers = package.searchers or package.loaders
  local off = 0
  if package.loaded[ "luarocks.loader" ] then off = 1 end
  assert( #searchers == 4+off, "package.searchers has been modified" )
  searchers[ 2+off ] = lua_searcher
end

-- Every loaded chunk is executed with a line and return hook enabled.
-- The line/return hook is responsible for collecting the test
-- functions.
for i,c in ipairs( chunks ) do
  -- `arg[0]` is set to the name of the loaded file to pretend as if
  -- the loaded file was executed by the standalone `lua` interpreter.
  -- This probably is unnecessary since usually only modules or
  -- specialized test scripts are tested using **Testy**, but some
  -- script might attempt to parse the `arg` table.
  _G.arg[ 0 ] = files[ i ]
  -- The monky-patched version of `assert` is made available here
  -- already in case the module code stores global functions in
  -- upvalues.
  _G.assert = _G_assert
  debug.sethook( line_ret_hook, "lr" )
  -- The chunk is called as if loaded by the `require` function: A
  -- (fake) module name and the file location are passed as
  -- parameters. Errors during loading of the module code are also
  -- considered fatal and thus terminate the testing session.
  c( "module.test", files[ i ] )
  debug.sethook()
end

-- After all module/test files have been loaded and executed, the debug
-- hooks should have collected all local test functions from the main
-- chunks of the given files. Now those test functions are called to
-- actually run the tests.
for _,t in ipairs( tests ) do
  -- A nice caption for the test function is derived from the function
  -- name by stripping the `test_` prefix and replacing all single
  -- underscores with spaces (multiple consecutive underscores lose
  -- one).
  if do_tap then
    fh:write( "# ", t.caption, " ('", t.source, "')\n" )
  else
    local headerlen = #t.caption + #t.source + #gap + 5
    fh:write( t.caption, " ('", t.source, "')" )
    if headerlen >= max_line then
      fh:write( "\n" )
    else
      fh:write( gap )
      cursor_pos = headerlen
    end
  end
  fh:flush()
  -- The modified `assert` function and the new `testy_assert` are
  -- made available to the test functions. This happens before every
  -- test in case some module author messes with them. If the
  -- `testy.extra` module is available, the functions from this
  -- module are added to the globals table as well.
  _G.assert = _G_assert
  _G.testy_assert = _G_testy_assert
  for k,v in pairs( extra ) do
    _G[ k ] = v
  end
  -- The test functions are called with `debug.traceback` as error
  -- message handler, so that unhandled errors in test functions can
  -- be reported with stack traces.
  local ok, msg = xpcall( t.func, debug.traceback )
  -- After each test function a new line is started no matter what
  -- output the `assert`s in the test function produced.
  if cursor_pos ~= 0 then
    fh:write( "\n" )
    cursor_pos = 0
  end
  if not ok then
    -- Unhandled errors are reported here, including stack traces.
    -- Unhandled errors are considered bugs and should be fixed as
    -- soon as possible, because they prevent the following test
    -- assertions in the same test function from executing.
    n_errors = n_errors + 1
    if do_tap then
      fh:write( "# [ERROR] test function '", t.name, "' died:\n# ",
                msg:gsub( "\n", "\n# " ), "\n" )
    else
      fh:write( "[ERROR] test function '", t.name, "' died:\n ",
                msg:gsub( "\n", "\n " ), "\n" )
    end
  else
    if not do_tap then
      -- In case there were test failures during the execution of this
      -- test function, the details of those failures are written now.
      -- For the TAP output the failure details were printed already.
      for _,f in ipairs( t ) do
        fh:write( "[FAIL] ", t.source, ":", f.line,
                  ": in function '", t.name, "'\n" )
        if f.reason then
          fh:write( "\t", f.reason:gsub( "\n\t?", "\n\t" ), "\n" )
        end
      end
    end
  end
  fh:flush()
end

if do_tap then
  -- For the TAP output the "test plan" is written out. Any unhandled
  -- error during the test run is considered a missing test.
  fh:write( "1..", n_tests+n_errors, "\n" )
else
  -- Finally, the combined test results are printed.
  fh:write( n_tests, " tests (", n_passed, " ok, ", n_tests-n_passed,
            " failed, ", n_errors, " errors)\n" )
end
fh:flush()
-- In case there were test failures or even unhandled errors in the
-- test functions, the `testy.lua` script exits with a non-zero
-- exit status. TAP consumers (`prove` at least) get suspicious on a
-- non-zero exit status, though, so we avoid that for normal failures.
if not do_tap and n_tests ~= n_passed or n_errors > 0 then
  os.exit( 1, true )
end