ref: d8fadd0cb30377c98f5f69b5c3603b436149a45d fastpeg/internal/bootstrap/scanner/scanner_test.go -rw-r--r-- 2.3 KiB View raw
                                                                                
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
package scanner

import (
	"testing"

	"git.sr.ht/~mna/fastpeg/internal/bootstrap/token"
	"github.com/stretchr/testify/require"
)

type tuple struct {
	tok token.Token
	lit string
}

func TestScanner_Tokens(t *testing.T) {
	input := `
abc "" 'def'
[] <- ←⟵=
| /     &
&{a} ${ _1 } @{ _-_ }
. : ; ! ? * + ()
"'x'" '"y"' [z]
`

	output := []tuple{
		{token.Identifier, "abc"},
		{token.Literal, `""`},
		{token.Literal, `'def'`},
		{token.CharClass, `[]`},
		{token.Arrow, `<-`},
		{token.Arrow, `←`},
		{token.Arrow, `⟵`},
		{token.Arrow, `=`},
		{token.Separator, `|`},
		{token.Separator, `/`},
		{token.Ampersand, ``},
		{token.PredCoderef, `a`},
		{token.StateCoderef, `_1`},
		{token.ActionCoderef, `_-_`},
		{token.Dot, ``},
		{token.Colon, ``},
		{token.Semicolon, ``},
		{token.Exclamation, ``},
		{token.Question, ``},
		{token.Star, ``},
		{token.Plus, ``},
		{token.Lparen, ``},
		{token.Rparen, ``},
		{token.Literal, `"'x'"`},
		{token.Literal, `'"y"'`},
		{token.CharClass, `[z]`},
	}

	var s Scanner
	result := scanAll(&s, input)
	require.Equal(t, output, result)
	require.NoError(t, s.Err())
}

func TestScanner_Escapes(t *testing.T) {
	input := `
"\a\b\f\n\r\t\v\\\""
'\a\b\f\n\r\t\v\\\''
"\x00\x10\xa1\xAF\xff"
'\x00\x10\xA1\xaf\xff'
[\x00\x10\xA1\xaf\xff]
"\u0123\uabcd\uEFef\U00045678"
'\u0123\uabcd\uEFef\U00045678'
[\u0123\uabcd\uEFef\U00045678]
[\a\b\f\n\r\t\v\\\]\[\-\^]
[\pC\PL\p{X}\P{Latin}]
`

	output := []tuple{
		{token.Literal, `"\a\b\f\n\r\t\v\\\""`},
		{token.Literal, `'\a\b\f\n\r\t\v\\\''`},
		{token.Literal, `"\x00\x10\xa1\xAF\xff"`},
		{token.Literal, `'\x00\x10\xA1\xaf\xff'`},
		{token.CharClass, `[\x00\x10\xA1\xaf\xff]`},
		{token.Literal, `"\u0123\uabcd\uEFef\U00045678"`},
		{token.Literal, `'\u0123\uabcd\uEFef\U00045678'`},
		{token.CharClass, `[\u0123\uabcd\uEFef\U00045678]`},
		{token.CharClass, `[\a\b\f\n\r\t\v\\\]\[\-\^]`},
		{token.CharClass, `[\pC\PL\p{X}\P{Latin}]`},
	}

	var s Scanner
	result := scanAll(&s, input)
	require.Equal(t, output, result)
	require.NoError(t, s.Err())
}

func scanAll(s *Scanner, input string) []tuple {
	var result []tuple

	fs := token.NewFileSet()
	f := fs.AddFile("test", -1, len(input))
	s.Init(f, []byte(input))
	for {
		_, tok, lit := s.Scan()
		if tok == token.EOF {
			break
		}
		result = append(result, tuple{tok, lit})
	}
	return result
}