~mna/snow

a69be6da04404f492c034b76008b59388c70d039 — Martin Angers 1 year, 10 months ago df88c58
big refactor of compiler package into sub-packages
114 files changed, 1116 insertions(+), 1065 deletions(-)

M .golangci.toml
D pkg/compiler/ast.go
A pkg/compiler/ast/ast.go
A pkg/compiler/ast/visitor.go
D pkg/compiler/codegen.go
A pkg/compiler/codegen/codegen.go
R pkg/compiler/{parser.go => parser/parser.go}
R pkg/compiler/{parser_test.go => parser/parser_test.go}
R pkg/compiler/{testdata/parser/comment.snow => parser/testdata/comment.snow}
R pkg/compiler/{testdata/parser/comment.snow.err => parser/testdata/comment.snow.err}
R pkg/compiler/{testdata/parser/comment.snow.want => parser/testdata/comment.snow.want}
R pkg/compiler/{testdata/parser/comments.snow => parser/testdata/comments.snow}
R pkg/compiler/{testdata/parser/comments.snow.err => parser/testdata/comments.snow.err}
R pkg/compiler/{testdata/parser/comments.snow.want => parser/testdata/comments.snow.want}
R pkg/compiler/{testdata/parser/empty.snow => parser/testdata/empty.snow}
R pkg/compiler/{testdata/parser/empty.snow.err => parser/testdata/empty.snow.err}
R pkg/compiler/{testdata/parser/empty.snow.want => parser/testdata/empty.snow.want}
R pkg/compiler/{testdata/parser/fn.snow => parser/testdata/fn.snow}
R pkg/compiler/{testdata/parser/fn.snow.err => parser/testdata/fn.snow.err}
R pkg/compiler/{testdata/parser/fn.snow.want => parser/testdata/fn.snow.want}
R pkg/compiler/{testdata/parser/fn_add.snow => parser/testdata/fn_add.snow}
R pkg/compiler/{testdata/parser/fn_add.snow.err => parser/testdata/fn_add.snow.err}
R pkg/compiler/{testdata/parser/fn_add.snow.want => parser/testdata/fn_add.snow.want}
R pkg/compiler/{testdata/parser/fn_assign.snow => parser/testdata/fn_assign.snow}
R pkg/compiler/{testdata/parser/fn_assign.snow.err => parser/testdata/fn_assign.snow.err}
R pkg/compiler/{testdata/parser/fn_assign.snow.want => parser/testdata/fn_assign.snow.want}
R pkg/compiler/{testdata/parser/fn_assign_invalid_lhs.snow => parser/testdata/fn_assign_invalid_lhs.snow}
R pkg/compiler/{testdata/parser/fn_assign_invalid_lhs.snow.err => parser/testdata/fn_assign_invalid_lhs.snow.err}
R pkg/compiler/{testdata/parser/fn_assign_invalid_lhs.snow.want => parser/testdata/fn_assign_invalid_lhs.snow.want}
R pkg/compiler/{testdata/parser/fn_attr_invalid.snow => parser/testdata/fn_attr_invalid.snow}
R pkg/compiler/{testdata/parser/fn_attr_invalid.snow.err => parser/testdata/fn_attr_invalid.snow.err}
R pkg/compiler/{testdata/parser/fn_attr_invalid.snow.want => parser/testdata/fn_attr_invalid.snow.want}
R pkg/compiler/{testdata/parser/fn_attr_many.snow => parser/testdata/fn_attr_many.snow}
R pkg/compiler/{testdata/parser/fn_attr_many.snow.err => parser/testdata/fn_attr_many.snow.err}
R pkg/compiler/{testdata/parser/fn_attr_many.snow.want => parser/testdata/fn_attr_many.snow.want}
R pkg/compiler/{testdata/parser/fn_expr_stmt.snow => parser/testdata/fn_expr_stmt.snow}
R pkg/compiler/{testdata/parser/fn_expr_stmt.snow.err => parser/testdata/fn_expr_stmt.snow.err}
R pkg/compiler/{testdata/parser/fn_expr_stmt.snow.want => parser/testdata/fn_expr_stmt.snow.want}
R pkg/compiler/{testdata/parser/fn_extern.snow => parser/testdata/fn_extern.snow}
R pkg/compiler/{testdata/parser/fn_extern.snow.err => parser/testdata/fn_extern.snow.err}
R pkg/compiler/{testdata/parser/fn_extern.snow.want => parser/testdata/fn_extern.snow.want}
R pkg/compiler/{testdata/parser/fn_mul.snow => parser/testdata/fn_mul.snow}
R pkg/compiler/{testdata/parser/fn_mul.snow.err => parser/testdata/fn_mul.snow.err}
R pkg/compiler/{testdata/parser/fn_mul.snow.want => parser/testdata/fn_mul.snow.want}
R pkg/compiler/{testdata/parser/fn_naked_return.snow => parser/testdata/fn_naked_return.snow}
R pkg/compiler/{testdata/parser/fn_naked_return.snow.err => parser/testdata/fn_naked_return.snow.err}
R pkg/compiler/{testdata/parser/fn_naked_return.snow.want => parser/testdata/fn_naked_return.snow.want}
R pkg/compiler/{testdata/parser/fn_nested_blocks.snow => parser/testdata/fn_nested_blocks.snow}
R pkg/compiler/{testdata/parser/fn_nested_blocks.snow.err => parser/testdata/fn_nested_blocks.snow.err}
R pkg/compiler/{testdata/parser/fn_nested_blocks.snow.want => parser/testdata/fn_nested_blocks.snow.want}
R pkg/compiler/{testdata/parser/fn_nested_paren.snow => parser/testdata/fn_nested_paren.snow}
R pkg/compiler/{testdata/parser/fn_nested_paren.snow.err => parser/testdata/fn_nested_paren.snow.err}
R pkg/compiler/{testdata/parser/fn_nested_paren.snow.want => parser/testdata/fn_nested_paren.snow.want}
R pkg/compiler/{testdata/parser/fn_nested_paren_newline.snow => parser/testdata/fn_nested_paren_newline.snow}
R pkg/compiler/{testdata/parser/fn_nested_paren_newline.snow.err => parser/testdata/fn_nested_paren_newline.snow.err}
R pkg/compiler/{testdata/parser/fn_nested_paren_newline.snow.want => parser/testdata/fn_nested_paren_newline.snow.want}
R pkg/compiler/{testdata/parser/fn_nested_unary.snow => parser/testdata/fn_nested_unary.snow}
R pkg/compiler/{testdata/parser/fn_nested_unary.snow.err => parser/testdata/fn_nested_unary.snow.err}
R pkg/compiler/{testdata/parser/fn_nested_unary.snow.want => parser/testdata/fn_nested_unary.snow.want}
R pkg/compiler/{testdata/parser/fn_paren_expr.snow => parser/testdata/fn_paren_expr.snow}
R pkg/compiler/{testdata/parser/fn_paren_expr.snow.err => parser/testdata/fn_paren_expr.snow.err}
R pkg/compiler/{testdata/parser/fn_paren_expr.snow.want => parser/testdata/fn_paren_expr.snow.want}
R pkg/compiler/{testdata/parser/fn_return_return.snow => parser/testdata/fn_return_return.snow}
R pkg/compiler/{testdata/parser/fn_return_return.snow.err => parser/testdata/fn_return_return.snow.err}
R pkg/compiler/{testdata/parser/fn_return_return.snow.want => parser/testdata/fn_return_return.snow.want}
R pkg/compiler/{testdata/parser/fn_unary.snow => parser/testdata/fn_unary.snow}
R pkg/compiler/{testdata/parser/fn_unary.snow.err => parser/testdata/fn_unary.snow.err}
R pkg/compiler/{testdata/parser/fn_unary.snow.want => parser/testdata/fn_unary.snow.want}
R pkg/compiler/{testdata/parser/let.snow => parser/testdata/let.snow}
R pkg/compiler/{testdata/parser/let.snow.err => parser/testdata/let.snow.err}
R pkg/compiler/{testdata/parser/let.snow.want => parser/testdata/let.snow.want}
R pkg/compiler/{testdata/parser/let_full.snow => parser/testdata/let_full.snow}
R pkg/compiler/{testdata/parser/let_full.snow.err => parser/testdata/let_full.snow.err}
R pkg/compiler/{testdata/parser/let_full.snow.want => parser/testdata/let_full.snow.want}
R pkg/compiler/{testdata/parser/let_infer.snow => parser/testdata/let_infer.snow}
R pkg/compiler/{testdata/parser/let_infer.snow.err => parser/testdata/let_infer.snow.err}
R pkg/compiler/{testdata/parser/let_infer.snow.want => parser/testdata/let_infer.snow.want}
R pkg/compiler/{testdata/parser/let_question.snow => parser/testdata/let_question.snow}
R pkg/compiler/{testdata/parser/let_question.snow.err => parser/testdata/let_question.snow.err}
R pkg/compiler/{testdata/parser/let_question.snow.want => parser/testdata/let_question.snow.want}
R pkg/compiler/{testdata/parser/return.snow => parser/testdata/return.snow}
R pkg/compiler/{testdata/parser/return.snow.err => parser/testdata/return.snow.err}
R pkg/compiler/{testdata/parser/return.snow.want => parser/testdata/return.snow.want}
R pkg/compiler/{testdata/parser/return_return.snow => parser/testdata/return_return.snow}
R pkg/compiler/{testdata/parser/return_return.snow.err => parser/testdata/return_return.snow.err}
R pkg/compiler/{testdata/parser/return_return.snow.want => parser/testdata/return_return.snow.want}
R pkg/compiler/{testdata/parser/var.snow => parser/testdata/var.snow}
R pkg/compiler/{testdata/parser/var.snow.err => parser/testdata/var.snow.err}
R pkg/compiler/{testdata/parser/var.snow.want => parser/testdata/var.snow.want}
R pkg/compiler/{testdata/parser/var_bang.snow => parser/testdata/var_bang.snow}
R pkg/compiler/{testdata/parser/var_bang.snow.err => parser/testdata/var_bang.snow.err}
R pkg/compiler/{testdata/parser/var_bang.snow.want => parser/testdata/var_bang.snow.want}
R pkg/compiler/{testdata/parser/var_comma_for_semi.snow => parser/testdata/var_comma_for_semi.snow}
R pkg/compiler/{testdata/parser/var_comma_for_semi.snow.err => parser/testdata/var_comma_for_semi.snow.err}
R pkg/compiler/{testdata/parser/var_comma_for_semi.snow.want => parser/testdata/var_comma_for_semi.snow.want}
R pkg/compiler/{testdata/parser/var_full.snow => parser/testdata/var_full.snow}
R pkg/compiler/{testdata/parser/var_full.snow.err => parser/testdata/var_full.snow.err}
R pkg/compiler/{testdata/parser/var_full.snow.want => parser/testdata/var_full.snow.want}
R pkg/compiler/{testdata/parser/var_infer.snow => parser/testdata/var_infer.snow}
R pkg/compiler/{testdata/parser/var_infer.snow.err => parser/testdata/var_infer.snow.err}
R pkg/compiler/{testdata/parser/var_infer.snow.want => parser/testdata/var_infer.snow.want}
R pkg/compiler/{testdata/parser/var_invalid_type.snow => parser/testdata/var_invalid_type.snow}
R pkg/compiler/{testdata/parser/var_invalid_type.snow.err => parser/testdata/var_invalid_type.snow.err}
R pkg/compiler/{testdata/parser/var_invalid_type.snow.want => parser/testdata/var_invalid_type.snow.want}
R pkg/compiler/{testdata/parser/var_missing_type_init.snow.notyet => parser/testdata/var_missing_type_init.snow.notyet}
R pkg/compiler/{printer.go => printer/printer.go}
A pkg/compiler/printer/printer_test.go
D pkg/compiler/printer_test.go
R pkg/compiler/{scanner.go => scanner/scanner.go}
R pkg/compiler/{scanner_test.go => scanner/scanner_test.go}
R pkg/compiler/testdata/{codegen/fn_add.snow => fn_add.snow}
R pkg/compiler/{token.go => token/token.go}
R pkg/compiler/{token_test.go => token/token_test.go}
D pkg/compiler/visitor.go
M .golangci.toml => .golangci.toml +13 -10
@@ 26,22 26,25 @@

[issues]
  # regexps of issue texts to exclude
  # NOTE: using this instead of [[issues.exclude-rules]] as SpaceVim respects those
  # exclusions, but does not respect the specific exclude-rules.
  exclude = [
    # package-level vars for binaries, assigned with linker flags
    # cmd/*: package-level vars for binaries, assigned with linker flags
    "`version` is a global variable",
    "`buildDate` is a global variable",
    # usage variables for command help, constant-like but built with fmt.Sprintf
    "`shortUsage` is a global variable",
    "`longUsage` is a global variable",
    "`commands` is a global variable",
  ]

[[issues.exclude-rules]]
  # slices of test cases in resp are globals because reused in multiple tests
  path = 'pkg/compiler/token\.go'
  linters = ["gochecknoglobals", "gochecknoinits"]
    # pkg/compiler/parser
    "`topLevelStmtStart` is a global variable",
    "`stmtStart` is a global variable",
    "`testUpdateParserTests` is a global variable",

[[issues.exclude-rules]]
  path = 'pkg/compiler/parser.*\.go'
  linters = ["gochecknoglobals"]
    # pkg/compiler/token
    "`NewFileSet` is a global variable",
    "`tokens` is a global variable",
    "`keywords` is a global variable",
    "`operators` is a global variable",
  ]


D pkg/compiler/ast.go => pkg/compiler/ast.go +0 -285
@@ 1,285 0,0 @@
package compiler

import (
	gotoken "go/token"
)

// represents a SourceFile production.
type file struct {
	stmts    []stmt
	comments []*commentGroup // for now, all comments found in the file end up here
}

func (f *file) Pos() gotoken.Pos {
	min := gotoken.NoPos
	if len(f.stmts) > 0 {
		min = f.stmts[0].Pos()
	}
	if len(f.comments) > 0 {
		if p := f.comments[0].Pos(); p < min || !min.IsValid() {
			min = p
		}
	}
	return min
}

func (f *file) End() gotoken.Pos {
	max := gotoken.NoPos
	if l := len(f.stmts); l > 0 {
		max = f.stmts[l-1].End()
	}
	if l := len(f.comments); l > 0 {
		if p := f.comments[l-1].End(); p > max || !max.IsValid() {
			max = p
		}
	}
	return max
}

// represents a FuncDef production.
type fnDef struct {
	attrs     []*attr // possibly empty
	fn        gotoken.Pos
	name      *ident
	signature *fnSig
	body      *block // possibly nil for external func declarations
}

func (f *fnDef) Pos() gotoken.Pos {
	if len(f.attrs) > 0 {
		return f.attrs[0].Pos()
	}
	return f.fn
}

func (f *fnDef) End() gotoken.Pos {
	if f.body == nil {
		return f.signature.End()
	}
	return f.body.End()
}

type fnSig struct {
	lparen gotoken.Pos // (
	params []*paramDef // possibly empty
	rparen gotoken.Pos // )
	rarrow gotoken.Pos // -> before return type, or gotoken.NoPos if no return
	retTyp *ident      // possibly nil
}

func (f *fnSig) Pos() gotoken.Pos { return f.lparen }
func (f *fnSig) End() gotoken.Pos {
	if !f.rarrow.IsValid() {
		return f.rparen + 1
	}
	return f.retTyp.End()
}

type paramDef struct {
	name  *ident
	colon gotoken.Pos
	typ   *ident
	comma gotoken.Pos // gotoken.NoPos if no trailing comma
}

func (p *paramDef) Pos() gotoken.Pos { return p.name.Pos() }
func (p *paramDef) End() gotoken.Pos {
	if p.comma.IsValid() {
		return p.comma + 1
	}
	return p.typ.End()
}

type block struct {
	lbrace gotoken.Pos
	stmts  []stmt
	rbrace gotoken.Pos
}

func (b *block) Pos() gotoken.Pos { return b.lbrace }
func (b *block) End() gotoken.Pos { return b.rbrace + 1 }

type attr struct {
	at     gotoken.Pos
	name   *ident
	lbrace gotoken.Pos
	fields []*keyValue
	rbrace gotoken.Pos
}

func (a *attr) Pos() gotoken.Pos { return a.at }
func (a *attr) End() gotoken.Pos { return a.rbrace + 1 }

type keyValue struct {
	key   *ident
	colon gotoken.Pos
	value *basicLit
	comma gotoken.Pos // gotoken.NoPos if no trailing comma
}

func (k *keyValue) Pos() gotoken.Pos { return k.key.Pos() }
func (k *keyValue) End() gotoken.Pos {
	if k.comma.IsValid() {
		return k.comma + 1
	}
	return k.value.End()
}

// represents a VarDef production.
type varDef struct {
	kw     token
	kwPos  gotoken.Pos
	name   *ident
	colon  gotoken.Pos // gotoken.NoPos if no colon (no explicit type)
	typ    *ident      // may be nil
	assign gotoken.Pos // gotoken.NoPos if no value assigned
	value  expr
}

func (v *varDef) Pos() gotoken.Pos { return v.kwPos }
func (v *varDef) End() gotoken.Pos {
	if v.value != nil {
		return v.value.End()
	}
	return v.typ.End()
}

// represents a ReturnStmt production.
type returnStmt struct {
	ret   gotoken.Pos
	value expr // may be nil for a naked return
}

func (r *returnStmt) Pos() gotoken.Pos { return r.ret }
func (r *returnStmt) End() gotoken.Pos {
	if r.value != nil {
		return r.value.End()
	}
	return gotoken.Pos(int(r.ret) + len(Return.String()))
}

// a binary expression, e.g. a * 2
type binaryExpr struct {
	left  expr
	op    token
	opPos gotoken.Pos
	right expr
}

func (b *binaryExpr) Pos() gotoken.Pos { return b.left.Pos() }
func (b *binaryExpr) End() gotoken.Pos { return b.right.End() }

// an unary expression, e.g. -4
type unaryExpr struct {
	op    token
	opPos gotoken.Pos
	right expr
}

func (u *unaryExpr) Pos() gotoken.Pos { return u.opPos }
func (u *unaryExpr) End() gotoken.Pos { return u.right.End() }

// a parenthesized expression e.g. ( a + 2 )
type parenExpr struct {
	lparen gotoken.Pos
	value  expr
	rparen gotoken.Pos
}

func (p *parenExpr) Pos() gotoken.Pos { return p.lparen }
func (p *parenExpr) End() gotoken.Pos { return p.rparen + 1 }

type assignStmt struct {
	left   expr
	assign gotoken.Pos
	right  expr
}

func (a *assignStmt) Pos() gotoken.Pos { return a.left.Pos() }
func (a *assignStmt) End() gotoken.Pos { return a.right.End() }

type exprStmt struct {
	value expr
}

func (e *exprStmt) Pos() gotoken.Pos { return e.value.Pos() }
func (e *exprStmt) End() gotoken.Pos { return e.value.End() }

// a basic literal, e.g. 33 or "abc"
type basicLit struct {
	pos     gotoken.Pos
	literal token // e.g. String or Int
	value   string
}

func (b *basicLit) Pos() gotoken.Pos { return b.pos }
func (b *basicLit) End() gotoken.Pos { return gotoken.Pos(int(b.pos) + len(b.value)) }

type ident struct {
	name string
	pos  gotoken.Pos
}

func (i *ident) Pos() gotoken.Pos { return i.pos }
func (i *ident) End() gotoken.Pos { return gotoken.Pos(int(i.pos) + len(i.name)) }

type commentGroup struct {
	list []*comment // len(List) > 0
}

func (g *commentGroup) Pos() gotoken.Pos { return g.list[0].Pos() }
func (g *commentGroup) End() gotoken.Pos { return g.list[len(g.list)-1].End() }

type comment struct {
	pound gotoken.Pos // Position of the starting '#'
	text  string      // including the '#', excluding the terminating '\n'
}

func (c *comment) Pos() gotoken.Pos { return c.pound }
func (c *comment) End() gotoken.Pos { return gotoken.Pos(int(c.pound) + len(c.text)) }

type badStmt struct {
	from, to gotoken.Pos
}

func (b *badStmt) Pos() gotoken.Pos { return b.from }
func (b *badStmt) End() gotoken.Pos { return b.to }

type badExpr struct {
	from, to gotoken.Pos
}

func (b *badExpr) Pos() gotoken.Pos { return b.from }
func (b *badExpr) End() gotoken.Pos { return b.to }

type node interface {
	Pos() gotoken.Pos // first character belonging to the node
	End() gotoken.Pos // first character immediately after the node
}

// expressions
type expr interface {
	node
	exprNode()
}

func (b *binaryExpr) exprNode() {}
func (u *unaryExpr) exprNode()  {}
func (p *parenExpr) exprNode()  {}
func (b *basicLit) exprNode()   {}
func (i *ident) exprNode()      {}
func (b *badExpr) exprNode()    {}

// statements
type stmt interface {
	node
	stmtNode()
}

func (f *fnDef) stmtNode()      {}
func (v *varDef) stmtNode()     {}
func (r *returnStmt) stmtNode() {}
func (b *badStmt) stmtNode()    {}
func (b *block) stmtNode()      {}
func (a *assignStmt) stmtNode() {}
func (e *exprStmt) stmtNode()   {}

A pkg/compiler/ast/ast.go => pkg/compiler/ast/ast.go +299 -0
@@ 0,0 1,299 @@
package ast

import (
	"git.sr.ht/~mna/snow/pkg/compiler/token"
)

// File represents a SourceFile production.
type File struct {
	Stmts    []Stmt
	Comments []*CommentGroup // for now, all comments found in the file end up here
}

func (f *File) Pos() token.Pos {
	min := token.NoPos
	if len(f.Stmts) > 0 {
		min = f.Stmts[0].Pos()
	}
	if len(f.Comments) > 0 {
		if p := f.Comments[0].Pos(); p < min || !min.IsValid() {
			min = p
		}
	}
	return min
}

func (f *File) End() token.Pos {
	max := token.NoPos
	if l := len(f.Stmts); l > 0 {
		max = f.Stmts[l-1].End()
	}
	if l := len(f.Comments); l > 0 {
		if p := f.Comments[l-1].End(); p > max || !max.IsValid() {
			max = p
		}
	}
	return max
}

// FnDef represents a FuncDef production.
type FnDef struct {
	Attrs     []*Attr // possibly empty
	Fn        token.Pos
	Name      *Ident
	Signature *FnSig
	Body      *Block // possibly nil for external func declarations
}

func (f *FnDef) Pos() token.Pos {
	if len(f.Attrs) > 0 {
		return f.Attrs[0].Pos()
	}
	return f.Fn
}

func (f *FnDef) End() token.Pos {
	if f.Body == nil {
		return f.Signature.End()
	}
	return f.Body.End()
}

// FnSig represents the signature part of a function definition
// (the parentheses with parameters and return type).
type FnSig struct {
	Lparen  token.Pos   // (
	Params  []*ParamDef // possibly empty
	Rparen  token.Pos   // )
	Rarrow  token.Pos   // -> before return type, or token.NoPos if no return
	RetType *Ident      // possibly nil
}

func (f *FnSig) Pos() token.Pos { return f.Lparen }
func (f *FnSig) End() token.Pos {
	if !f.Rarrow.IsValid() {
		return f.Rparen + 1
	}
	return f.RetType.End()
}

// ParamDef represents a parameter definition in a function signature.
type ParamDef struct {
	Name  *Ident
	Colon token.Pos
	Type  *Ident
	Comma token.Pos // token.NoPos if no trailing comma
}

func (p *ParamDef) Pos() token.Pos { return p.Name.Pos() }
func (p *ParamDef) End() token.Pos {
	if p.Comma.IsValid() {
		return p.Comma + 1
	}
	return p.Type.End()
}

// Block represents a block of statements, including the surrounding braces.
type Block struct {
	Lbrace token.Pos
	Stmts  []Stmt
	Rbrace token.Pos
}

func (b *Block) Pos() token.Pos { return b.Lbrace }
func (b *Block) End() token.Pos { return b.Rbrace + 1 }

// Attr represents a function attribute.
type Attr struct {
	At     token.Pos
	Name   *Ident
	Lbrace token.Pos
	Fields []*KeyValue
	Rbrace token.Pos
}

func (a *Attr) Pos() token.Pos { return a.At }
func (a *Attr) End() token.Pos { return a.Rbrace + 1 }

// KeyValue represents a key-value pair in an Attr.
type KeyValue struct {
	Key   *Ident
	Colon token.Pos
	Value *BasicLit
	Comma token.Pos // token.NoPos if no trailing comma
}

func (k *KeyValue) Pos() token.Pos { return k.Key.Pos() }
func (k *KeyValue) End() token.Pos {
	if k.Comma.IsValid() {
		return k.Comma + 1
	}
	return k.Value.End()
}

// VarDef represents a VarDef production.
type VarDef struct {
	Kw     token.Token
	KwPos  token.Pos
	Name   *Ident
	Colon  token.Pos // token.NoPos if no colon (no explicit type)
	Type   *Ident    // may be nil
	Assign token.Pos // token.NoPos if no value assigned
	Value  Expr
}

func (v *VarDef) Pos() token.Pos { return v.KwPos }
func (v *VarDef) End() token.Pos {
	if v.Value != nil {
		return v.Value.End()
	}
	return v.Type.End()
}

// ReturnStmt represents a ReturnStmt production.
type ReturnStmt struct {
	Ret   token.Pos
	Value Expr // may be nil for a naked return
}

func (r *ReturnStmt) Pos() token.Pos { return r.Ret }
func (r *ReturnStmt) End() token.Pos {
	if r.Value != nil {
		return r.Value.End()
	}
	return token.Pos(int(r.Ret) + len(token.Return.String()))
}

// BinaryExpr represents a binary expression, e.g. a * 2.
type BinaryExpr struct {
	Left  Expr
	Op    token.Token
	OpPos token.Pos
	Right Expr
}

func (b *BinaryExpr) Pos() token.Pos { return b.Left.Pos() }
func (b *BinaryExpr) End() token.Pos { return b.Right.End() }

// UnaryExpr represents an unary expression, e.g. -4.
type UnaryExpr struct {
	Op    token.Token
	OpPos token.Pos
	Right Expr
}

func (u *UnaryExpr) Pos() token.Pos { return u.OpPos }
func (u *UnaryExpr) End() token.Pos { return u.Right.End() }

// ParenExpr represents a parenthesized expression e.g. ( a + 2 ).
type ParenExpr struct {
	Lparen token.Pos
	Value  Expr
	Rparen token.Pos
}

func (p *ParenExpr) Pos() token.Pos { return p.Lparen }
func (p *ParenExpr) End() token.Pos { return p.Rparen + 1 }

// AssignStmt represents an assignment statement, e.g. x = y + z.
type AssignStmt struct {
	Left   Expr
	Assign token.Pos
	Right  Expr
}

func (a *AssignStmt) Pos() token.Pos { return a.Left.Pos() }
func (a *AssignStmt) End() token.Pos { return a.Right.End() }

// ExprStmt represents a standalone expression statement, e.g. 3 or x.
type ExprStmt struct {
	Value Expr
}

func (e *ExprStmt) Pos() token.Pos { return e.Value.Pos() }
func (e *ExprStmt) End() token.Pos { return e.Value.End() }

// BasicLit represents a basic literal, e.g. 33 or "abc".
type BasicLit struct {
	LitPos  token.Pos
	Literal token.Token // e.g. String or Int
	Value   string
}

func (b *BasicLit) Pos() token.Pos { return b.LitPos }
func (b *BasicLit) End() token.Pos { return token.Pos(int(b.LitPos) + len(b.Value)) }

// Ident represents an identifier.
type Ident struct {
	Name     string
	IdentPos token.Pos
}

func (i *Ident) Pos() token.Pos { return i.IdentPos }
func (i *Ident) End() token.Pos { return token.Pos(int(i.IdentPos) + len(i.Name)) }

// CommentGroup represents a sequence of comments.
type CommentGroup struct {
	List []*Comment // len(List) > 0
}

func (g *CommentGroup) Pos() token.Pos { return g.List[0].Pos() }
func (g *CommentGroup) End() token.Pos { return g.List[len(g.List)-1].End() }

// Comment represents a single comment.
type Comment struct {
	Pound token.Pos // Position of the starting '#'
	Text  string    // including the '#', excluding the terminating '\n'
}

func (c *Comment) Pos() token.Pos { return c.Pound }
func (c *Comment) End() token.Pos { return token.Pos(int(c.Pound) + len(c.Text)) }

// BadStmt represents a bad (invalid) statement.
type BadStmt struct {
	From, To token.Pos
}

func (b *BadStmt) Pos() token.Pos { return b.From }
func (b *BadStmt) End() token.Pos { return b.To }

// BadExpr represents a bad (invalid) expression.
type BadExpr struct {
	From, To token.Pos
}

func (b *BadExpr) Pos() token.Pos { return b.From }
func (b *BadExpr) End() token.Pos { return b.To }

// Node represents a node in the AST.
type Node interface {
	Pos() token.Pos // first character belonging to the node
	End() token.Pos // first character immediately after the node
}

// Expr represents an expression in the AST.
type Expr interface {
	Node
	exprNode()
}

func (b *BinaryExpr) exprNode() {}
func (u *UnaryExpr) exprNode()  {}
func (p *ParenExpr) exprNode()  {}
func (b *BasicLit) exprNode()   {}
func (i *Ident) exprNode()      {}
func (b *BadExpr) exprNode()    {}

// Stmt represents a statement in the AST.
type Stmt interface {
	Node
	stmtNode()
}

func (f *FnDef) stmtNode()      {}
func (v *VarDef) stmtNode()     {}
func (r *ReturnStmt) stmtNode() {}
func (b *BadStmt) stmtNode()    {}
func (b *Block) stmtNode()      {}
func (a *AssignStmt) stmtNode() {}
func (e *ExprStmt) stmtNode()   {}

A pkg/compiler/ast/visitor.go => pkg/compiler/ast/visitor.go +133 -0
@@ 0,0 1,133 @@
package ast

import "fmt"

type Visitor interface {
	Visit(n Node) (w Visitor)
}

func Walk(v Visitor, node Node) {
	if v = v.Visit(node); v == nil {
		return
	}

	switch n := node.(type) {
	case *File:
		for _, cg := range n.Comments {
			Walk(v, cg)
		}
		for _, stmt := range n.Stmts {
			Walk(v, stmt)
		}

	case *FnDef:
		for _, attr := range n.Attrs {
			Walk(v, attr)
		}
		if n.Name != nil {
			Walk(v, n.Name)
		}
		if n.Signature != nil {
			Walk(v, n.Signature)
		}
		if n.Body != nil {
			Walk(v, n.Body)
		}

	case *FnSig:
		for _, param := range n.Params {
			Walk(v, param)
		}
		if n.RetType != nil {
			Walk(v, n.RetType)
		}

	case *ParamDef:
		if n.Name != nil {
			Walk(v, n.Name)
		}
		if n.Type != nil {
			Walk(v, n.Type)
		}

	case *Block:
		for _, stmt := range n.Stmts {
			Walk(v, stmt)
		}

	case *Attr:
		if n.Name != nil {
			Walk(v, n.Name)
		}
		for _, field := range n.Fields {
			Walk(v, field)
		}

	case *KeyValue:
		if n.Key != nil {
			Walk(v, n.Key)
		}
		if n.Value != nil {
			Walk(v, n.Value)
		}

	case *VarDef:
		if n.Name != nil {
			Walk(v, n.Name)
		}
		if n.Type != nil {
			Walk(v, n.Type)
		}
		if n.Value != nil {
			Walk(v, n.Value)
		}

	case *ReturnStmt:
		if n.Value != nil {
			Walk(v, n.Value)
		}

	case *BinaryExpr:
		if n.Left != nil {
			Walk(v, n.Left)
		}
		if n.Right != nil {
			Walk(v, n.Right)
		}

	case *UnaryExpr:
		if n.Right != nil {
			Walk(v, n.Right)
		}

	case *ParenExpr:
		if n.Value != nil {
			Walk(v, n.Value)
		}

	case *AssignStmt:
		if n.Left != nil {
			Walk(v, n.Left)
		}
		if n.Right != nil {
			Walk(v, n.Right)
		}

	case *ExprStmt:
		if n.Value != nil {
			Walk(v, n.Value)
		}

	case *CommentGroup:
		for _, comment := range n.List {
			Walk(v, comment)
		}

	case *BasicLit, *Ident, *Comment, *BadStmt, *BadExpr:
		// no child to step into

	default:
		panic(fmt.Sprintf("Walk: unexpected node type %T", n))
	}
	v.Visit(nil)
}

D pkg/compiler/codegen.go => pkg/compiler/codegen.go +0 -1
@@ 1,1 0,0 @@
package compiler

A pkg/compiler/codegen/codegen.go => pkg/compiler/codegen/codegen.go +1 -0
@@ 0,0 1,1 @@
package codegen

R pkg/compiler/parser.go => pkg/compiler/parser/parser.go +177 -175
@@ 1,80 1,82 @@
package compiler
package parser

import (
	goscanner "go/scanner"
	gotoken "go/token"
	"git.sr.ht/~mna/snow/pkg/compiler/ast"
	"git.sr.ht/~mna/snow/pkg/compiler/scanner"
	"git.sr.ht/~mna/snow/pkg/compiler/token"
)

type parser struct {
	scanner scanner
	errors  goscanner.ErrorList
	file    *gotoken.File
// Parser parses source files and generates an AST.
type Parser struct {
	scanner scanner.Scanner
	errors  scanner.ErrorList
	file    *token.File

	// current token
	tok token
	pos gotoken.Pos
	tok token.Token
	pos token.Pos
	lit string

	// collected comments
	cgs []*commentGroup
	cgs []*ast.CommentGroup
}

func (p *parser) init(fset *gotoken.FileSet, filename string, src []byte) {
func (p *Parser) init(fset *token.FileSet, filename string, src []byte) {
	p.file = fset.AddFile(filename, -1, len(src))
	errHandler := func(pos gotoken.Position, msg string) { p.errors.Add(pos, msg) }
	p.scanner.init(p.file, src, errHandler)
	errHandler := func(pos token.Position, msg string) { p.errors.Add(pos, msg) }
	p.scanner.Init(p.file, src, errHandler)
	p.cgs = nil

	// advance to first token
	p.next()
}

func (p *parser) next() {
	var cg commentGroup
func (p *Parser) next() {
	var cg ast.CommentGroup

	p.pos, p.tok, p.lit = p.scanner.scan()
	for p.tok == Comment {
		cg.list = append(cg.list, &comment{pound: p.pos, text: p.lit})
		p.pos, p.tok, p.lit = p.scanner.scan()
	p.pos, p.tok, p.lit = p.scanner.Scan()
	for p.tok == token.Comment {
		cg.List = append(cg.List, &ast.Comment{Pound: p.pos, Text: p.lit})
		p.pos, p.tok, p.lit = p.scanner.Scan()
	}

	if len(cg.list) > 0 {
	if len(cg.List) > 0 {
		p.cgs = append(p.cgs, &cg)
	}
}

func (p *parser) parseFile() *file {
	var stmts []stmt
	for p.tok != EOF {
func (p *Parser) parseFile() *ast.File {
	var stmts []ast.Stmt
	for p.tok != token.EOF {
		stmts = append(stmts, p.parseStmt(true))
	}

	return &file{
		comments: p.cgs,
		stmts:    stmts,
	return &ast.File{
		Comments: p.cgs,
		Stmts:    stmts,
	}
}

func (p *parser) parseStmt(topLevel bool) stmt {
func (p *Parser) parseStmt(topLevel bool) ast.Stmt {
	if !topLevel {
		switch p.tok {
		case Return:
		case token.Return:
			return p.parseReturnStmt()
		case Lbrace:
		case token.Lbrace:
			bk := p.parseBlock()
			p.expectSemi()
			return bk

			// tokens that may start a simple statement
		case Ident, Add, Sub, Lparen, Int, String:
		case token.Ident, token.Add, token.Sub, token.Lparen, token.Int, token.String:
			return p.parseSimpleStmt()
		}
	}

	switch p.tok {
	case Let, Var:
	case token.Let, token.Var:
		return p.parseVarDef()
	case At, Fn:
	case token.At, token.Fn:
		return p.parseFuncDef()
	default:
		pos := p.pos


@@ 86,202 88,202 @@ func (p *parser) parseStmt(topLevel bool) stmt {
		}
		p.errorExpected(pos, lbl)
		p.advance(sync)
		return &badStmt{from: pos, to: p.pos}
		return &ast.BadStmt{From: pos, To: p.pos}
	}
}

func (p *parser) parseSimpleStmt() stmt {
func (p *Parser) parseSimpleStmt() ast.Stmt {
	pos := p.pos
	lhs := p.parseExpr()

	// check if this is an assignment statement
	if p.tok == Assign {
		opPos := p.expect(Assign)
	if p.tok == token.Assign {
		opPos := p.expect(token.Assign)

		// for now, lhs must be an identifier, otherwise convert to a badExpr
		if _, ok := lhs.(*ident); !ok {
		if _, ok := lhs.(*ast.Ident); !ok {
			p.errorExpected(pos, "identifier")
			lhs = &badExpr{
				from: lhs.Pos(),
				to:   lhs.End(),
			lhs = &ast.BadExpr{
				From: lhs.Pos(),
				To:   lhs.End(),
			}
		}

		as := &assignStmt{
			left:   lhs,
			assign: opPos,
		as := &ast.AssignStmt{
			Left:   lhs,
			Assign: opPos,
		}
		as.right = p.parseExpr()
		as.Right = p.parseExpr()
		p.expectSemi()
		return as
	}

	// otherwise this is an expression statement
	es := &exprStmt{value: lhs}
	es := &ast.ExprStmt{Value: lhs}
	p.expectSemi()
	return es
}

func (p *parser) parseReturnStmt() *returnStmt {
	rs := &returnStmt{
		ret: p.expect(Return),
func (p *Parser) parseReturnStmt() *ast.ReturnStmt {
	rs := &ast.ReturnStmt{
		Ret: p.expect(token.Return),
	}
	// != Rbrace so that `{ return }` is allowed on the same line
	if p.tok != Semicolon && p.tok != Rbrace {
		rs.value = p.parseExpr()
	if p.tok != token.Semicolon && p.tok != token.Rbrace {
		rs.Value = p.parseExpr()
	}
	p.expectSemi()
	return rs
}

func (p *parser) parseVarDef() *varDef {
	vd := &varDef{
		kw:    p.tok,
		kwPos: p.pos,
func (p *Parser) parseVarDef() *ast.VarDef {
	vd := &ast.VarDef{
		Kw:    p.tok,
		KwPos: p.pos,
	}

	pos := p.pos
	p.next()
	vd.name = p.parseIdent()
	vd.Name = p.parseIdent()

	if p.tok == Colon {
		vd.colon = p.pos
	if p.tok == token.Colon {
		vd.Colon = p.pos
		p.next()
		vd.typ = p.parseIdent()
		vd.Type = p.parseIdent()
	}
	if p.tok == Assign {
		vd.assign = p.pos
	if p.tok == token.Assign {
		vd.Assign = p.pos
		p.next()
		vd.value = p.parseExpr()
		vd.Value = p.parseExpr()
	}
	p.expectSemi()

	if vd.typ == nil && vd.value == nil {
	if vd.Type == nil && vd.Value == nil {
		p.error(pos, "missing variable type or initialization")
	}
	return vd
}

func (p *parser) parseIdent() *ident {
func (p *Parser) parseIdent() *ast.Ident {
	var name string
	if p.tok == Ident {
	if p.tok == token.Ident {
		name = p.lit
	}
	return &ident{
		name: name,
		pos:  p.expect(Ident),
	return &ast.Ident{
		Name:     name,
		IdentPos: p.expect(token.Ident),
	}
}

func (p *parser) parseBasicLit() *basicLit {
func (p *Parser) parseBasicLit() *ast.BasicLit {
	var lit string
	tok := Illegal
	if p.tok == String || p.tok == Int {
	tok := token.Illegal
	if p.tok == token.String || p.tok == token.Int {
		lit = p.lit
		tok = p.tok
	}
	return &basicLit{
		literal: tok,
		pos:     p.expect(String, Int),
		value:   lit,
	return &ast.BasicLit{
		Literal: tok,
		LitPos:  p.expect(token.String, token.Int),
		Value:   lit,
	}
}

func (p *parser) parseExpr() expr {
func (p *Parser) parseExpr() ast.Expr {
	lhs := p.parseTerm()
	for p.tok == Add || p.tok == Sub {
		bin := &binaryExpr{
			op:    p.tok,
			opPos: p.pos,
	for p.tok == token.Add || p.tok == token.Sub {
		bin := &ast.BinaryExpr{
			Op:    p.tok,
			OpPos: p.pos,
		}
		p.next()

		rhs := p.parseTerm()
		bin.left = lhs
		bin.right = rhs
		bin.Left = lhs
		bin.Right = rhs
		lhs = bin
	}
	return lhs
}

func (p *parser) parseTerm() expr {
func (p *Parser) parseTerm() ast.Expr {
	lhs := p.parseFactor()
	for p.tok == Mul || p.tok == Div || p.tok == Mod {
		bin := &binaryExpr{
			op:    p.tok,
			opPos: p.pos,
	for p.tok == token.Mul || p.tok == token.Div || p.tok == token.Mod {
		bin := &ast.BinaryExpr{
			Op:    p.tok,
			OpPos: p.pos,
		}
		p.next()

		rhs := p.parseFactor()
		bin.left = lhs
		bin.right = rhs
		bin.Left = lhs
		bin.Right = rhs
		lhs = bin
	}
	return lhs
}

func (p *parser) parseFactor() expr {
	if p.tok == Add || p.tok == Sub {
		un := &unaryExpr{
			op:    p.tok,
			opPos: p.pos,
func (p *Parser) parseFactor() ast.Expr {
	if p.tok == token.Add || p.tok == token.Sub {
		un := &ast.UnaryExpr{
			Op:    p.tok,
			OpPos: p.pos,
		}
		p.next()
		un.right = p.parseFactor()
		un.Right = p.parseFactor()
		return un
	}

	return p.parseAtom()
}

func (p *parser) parseAtom() expr {
func (p *Parser) parseAtom() ast.Expr {
	switch p.tok {
	case Lparen:
		pe := &parenExpr{
			lparen: p.pos,
	case token.Lparen:
		pe := &ast.ParenExpr{
			Lparen: p.pos,
		}
		p.next()
		pe.value = p.parseExpr()
		pe.rparen = p.expect(Rparen)
		pe.Value = p.parseExpr()
		pe.Rparen = p.expect(token.Rparen)
		return pe

	case Ident:
	case token.Ident:
		return p.parseIdent()
	case String, Int:
	case token.String, token.Int:
		return p.parseBasicLit()

	default:
		pos := p.pos
		p.errorExpected(pos, "expression")
		return &badExpr{from: pos, to: p.pos}
		return &ast.BadExpr{From: pos, To: p.pos}
	}
}

func (p *parser) parseAttr() *attr {
	a := &attr{
		at: p.expect(At),
func (p *Parser) parseAttr() *ast.Attr {
	a := &ast.Attr{
		At: p.expect(token.At),
	}
	a.name = p.parseIdent()
	a.lbrace = p.expect(Lbrace)
	a.fields = p.parseKeyValueList()
	a.rbrace = p.expect(Rbrace)
	a.Name = p.parseIdent()
	a.Lbrace = p.expect(token.Lbrace)
	a.Fields = p.parseKeyValueList()
	a.Rbrace = p.expect(token.Rbrace)
	p.expectSemi()
	return a
}

func (p *parser) parseKeyValueList() []*keyValue {
	var list []*keyValue
	for p.tok != Rbrace && p.tok != EOF {
		kv := &keyValue{
			key: p.parseIdent(),
func (p *Parser) parseKeyValueList() []*ast.KeyValue {
	var list []*ast.KeyValue
	for p.tok != token.Rbrace && p.tok != token.EOF {
		kv := &ast.KeyValue{
			Key: p.parseIdent(),
		}
		kv.colon = p.expect(Colon)
		kv.value = p.parseBasicLit()
		kv.Colon = p.expect(token.Colon)
		kv.Value = p.parseBasicLit()
		list = append(list, kv)

		if p.tok == Comma {
			kv.comma = p.expect(Comma)
		if p.tok == token.Comma {
			kv.Comma = p.expect(token.Comma)
		} else {
			break
		}


@@ 289,78 291,78 @@ func (p *parser) parseKeyValueList() []*keyValue {
	return list
}

func (p *parser) parseFuncDef() *fnDef {
	var attrs []*attr
	for p.tok == At {
func (p *Parser) parseFuncDef() *ast.FnDef {
	var attrs []*ast.Attr
	for p.tok == token.At {
		attrs = append(attrs, p.parseAttr())
	}

	fd := &fnDef{
		attrs: attrs,
		fn:    p.pos,
	fd := &ast.FnDef{
		Attrs: attrs,
		Fn:    p.pos,
	}

	p.next()
	fd.name = p.parseIdent()
	fd.signature = p.parseFuncSig()
	fd.Name = p.parseIdent()
	fd.Signature = p.parseFuncSig()

	// stop here if this is a function declaration (no body)
	if p.tok != Lbrace {
	if p.tok != token.Lbrace {
		p.expectSemi()
		return fd
	}

	// continue with function body (function definition)
	fd.body = p.parseBlock()
	fd.Body = p.parseBlock()
	p.expectSemi()
	return fd
}

func (p *parser) parseBlock() *block {
	lbrace := p.expect(Lbrace)
func (p *Parser) parseBlock() *ast.Block {
	lbrace := p.expect(token.Lbrace)
	stmts := p.parseStmtList()
	rbrace := p.expect(Rbrace)
	return &block{
		lbrace: lbrace,
		stmts:  stmts,
		rbrace: rbrace,
	rbrace := p.expect(token.Rbrace)
	return &ast.Block{
		Lbrace: lbrace,
		Stmts:  stmts,
		Rbrace: rbrace,
	}
}

func (p *parser) parseStmtList() []stmt {
	var list []stmt
	for p.tok != Rbrace && p.tok != EOF {
func (p *Parser) parseStmtList() []ast.Stmt {
	var list []ast.Stmt
	for p.tok != token.Rbrace && p.tok != token.EOF {
		list = append(list, p.parseStmt(false))
	}
	return list
}

func (p *parser) parseFuncSig() *fnSig {
	sig := &fnSig{
		lparen: p.expect(Lparen),
func (p *Parser) parseFuncSig() *ast.FnSig {
	sig := &ast.FnSig{
		Lparen: p.expect(token.Lparen),
	}
	sig.params = p.parseParamsList()
	sig.rparen = p.expect(Rparen)
	sig.Params = p.parseParamsList()
	sig.Rparen = p.expect(token.Rparen)

	if p.tok == Rarrow {
		sig.rarrow = p.expect(Rarrow)
		sig.retTyp = p.parseIdent()
	if p.tok == token.Rarrow {
		sig.Rarrow = p.expect(token.Rarrow)
		sig.RetType = p.parseIdent()
	}
	return sig
}

func (p *parser) parseParamsList() []*paramDef {
	var params []*paramDef
	for p.tok != Rparen && p.tok != EOF {
		arg := &paramDef{
			name: p.parseIdent(),
func (p *Parser) parseParamsList() []*ast.ParamDef {
	var params []*ast.ParamDef
	for p.tok != token.Rparen && p.tok != token.EOF {
		arg := &ast.ParamDef{
			Name: p.parseIdent(),
		}
		arg.colon = p.expect(Colon)
		arg.typ = p.parseIdent()
		arg.Colon = p.expect(token.Colon)
		arg.Type = p.parseIdent()
		params = append(params, arg)

		if p.tok == Comma {
			arg.comma = p.expect(Comma)
		if p.tok == token.Comma {
			arg.Comma = p.expect(token.Comma)
		} else {
			break
		}


@@ 368,18 370,18 @@ func (p *parser) parseParamsList() []*paramDef {
	return params
}

func (p *parser) error(pos gotoken.Pos, msg string) {
func (p *Parser) error(pos token.Pos, msg string) {
	lpos := p.file.Position(pos)
	p.errors.Add(lpos, msg)
}

func (p *parser) errorExpected(pos gotoken.Pos, msg string) {
func (p *Parser) errorExpected(pos token.Pos, msg string) {
	msg = "expected " + msg
	if pos == p.pos {
		// the error happened at the current position;
		// make the error message more specific
		switch {
		case p.tok.literal():
		case p.tok.Literal():
			// print 123 rather than 'INT', etc.
			msg += ", found " + p.lit
		default:


@@ 389,7 391,7 @@ func (p *parser) errorExpected(pos gotoken.Pos, msg string) {
	p.error(pos, msg)
}

func (p *parser) expect(toks ...token) gotoken.Pos {
func (p *Parser) expect(toks ...token.Token) token.Pos {
	pos := p.pos

	var lbl string


@@ 414,15 416,15 @@ func (p *parser) expect(toks ...token) gotoken.Pos {
	return pos
}

func (p *parser) expectSemi() {
func (p *Parser) expectSemi() {
	// semicolon is optional before a closing ')' or '}'
	if p.tok != Rparen && p.tok != Rbrace {
	if p.tok != token.Rparen && p.tok != token.Rbrace {
		switch p.tok {
		case Comma:
		case token.Comma:
			// properly parse a ',' instead of a ';' but add error
			p.errorExpected(p.pos, "';'")
			fallthrough
		case Semicolon:
		case token.Semicolon:
			p.next()
		default:
			p.errorExpected(p.pos, "';'")


@@ 434,22 436,22 @@ func (p *parser) expectSemi() {
// sets of tokens that can be sync'd to (in a call to p.advance)
var (
	// TopLevelStmt production in the grammar.
	topLevelStmtStart = map[token]bool{
		Var: true,
		Let: true,
		Fn:  true,
	topLevelStmtStart = map[token.Token]bool{
		token.Var: true,
		token.Let: true,
		token.Fn:  true,
	}

	stmtStart = map[token]bool{
		Var:    true,
		Let:    true,
		Fn:     true,
		Return: true,
	stmtStart = map[token.Token]bool{
		token.Var:    true,
		token.Let:    true,
		token.Fn:     true,
		token.Return: true,
	}
)

func (p *parser) advance(to map[token]bool) {
	for p.tok != EOF {
func (p *Parser) advance(to map[token.Token]bool) {
	for p.tok != token.EOF {
		if to[p.tok] {
			return
		}

R pkg/compiler/parser_test.go => pkg/compiler/parser/parser_test.go +5 -4
@@ 1,4 1,4 @@
package compiler
package parser

import (
	"bytes"


@@ 10,13 10,14 @@ import (
	"path/filepath"
	"testing"

	"git.sr.ht/~mna/snow/pkg/compiler/printer"
	"github.com/kylelemons/godebug/diff"
)

var testUpdateParserTests = flag.Bool("test.update-parser-tests", false, "If set, replace expected parser test results with actual results.")

func TestParser(t *testing.T) {
	baseDir := filepath.Join("testdata", "parser")
	baseDir := "testdata"
	fis, err := ioutil.ReadDir(baseDir)
	if err != nil {
		t.Fatal(err)


@@ 31,7 32,7 @@ func TestParser(t *testing.T) {
		}

		t.Run(fi.Name(), func(t *testing.T) {
			var p parser
			var p Parser

			b, err := ioutil.ReadFile(filepath.Join(baseDir, fi.Name()))
			if err != nil {


@@ 51,7 52,7 @@ func TestParser(t *testing.T) {
			goscanner.PrintError(&ebuf, parseErr)

			var buf bytes.Buffer
			pp := printer{w: &buf, posMode: posOffsets, withComments: true}
			pp := printer.Printer{W: &buf, Pos: printer.PosOffsets, WithComments: true}
			if err := pp.Print(f, p.file); err != nil {
				t.Fatal(err)
			}

R pkg/compiler/testdata/parser/comment.snow => pkg/compiler/parser/testdata/comment.snow +0 -0
R pkg/compiler/testdata/parser/comment.snow.err => pkg/compiler/parser/testdata/comment.snow.err +0 -0
R pkg/compiler/testdata/parser/comment.snow.want => pkg/compiler/parser/testdata/comment.snow.want +0 -0
R pkg/compiler/testdata/parser/comments.snow => pkg/compiler/parser/testdata/comments.snow +0 -0
R pkg/compiler/testdata/parser/comments.snow.err => pkg/compiler/parser/testdata/comments.snow.err +0 -0
R pkg/compiler/testdata/parser/comments.snow.want => pkg/compiler/parser/testdata/comments.snow.want +0 -0
R pkg/compiler/testdata/parser/empty.snow => pkg/compiler/parser/testdata/empty.snow +0 -0
R pkg/compiler/testdata/parser/empty.snow.err => pkg/compiler/parser/testdata/empty.snow.err +0 -0
R pkg/compiler/testdata/parser/empty.snow.want => pkg/compiler/parser/testdata/empty.snow.want +0 -0
R pkg/compiler/testdata/parser/fn.snow => pkg/compiler/parser/testdata/fn.snow +0 -0
R pkg/compiler/testdata/parser/fn.snow.err => pkg/compiler/parser/testdata/fn.snow.err +0 -0
R pkg/compiler/testdata/parser/fn.snow.want => pkg/compiler/parser/testdata/fn.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_add.snow => pkg/compiler/parser/testdata/fn_add.snow +0 -0
R pkg/compiler/testdata/parser/fn_add.snow.err => pkg/compiler/parser/testdata/fn_add.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_add.snow.want => pkg/compiler/parser/testdata/fn_add.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_assign.snow => pkg/compiler/parser/testdata/fn_assign.snow +0 -0
R pkg/compiler/testdata/parser/fn_assign.snow.err => pkg/compiler/parser/testdata/fn_assign.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_assign.snow.want => pkg/compiler/parser/testdata/fn_assign.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow => pkg/compiler/parser/testdata/fn_assign_invalid_lhs.snow +0 -0
R pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.err => pkg/compiler/parser/testdata/fn_assign_invalid_lhs.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.want => pkg/compiler/parser/testdata/fn_assign_invalid_lhs.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_attr_invalid.snow => pkg/compiler/parser/testdata/fn_attr_invalid.snow +0 -0
R pkg/compiler/testdata/parser/fn_attr_invalid.snow.err => pkg/compiler/parser/testdata/fn_attr_invalid.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_attr_invalid.snow.want => pkg/compiler/parser/testdata/fn_attr_invalid.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_attr_many.snow => pkg/compiler/parser/testdata/fn_attr_many.snow +0 -0
R pkg/compiler/testdata/parser/fn_attr_many.snow.err => pkg/compiler/parser/testdata/fn_attr_many.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_attr_many.snow.want => pkg/compiler/parser/testdata/fn_attr_many.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_expr_stmt.snow => pkg/compiler/parser/testdata/fn_expr_stmt.snow +0 -0
R pkg/compiler/testdata/parser/fn_expr_stmt.snow.err => pkg/compiler/parser/testdata/fn_expr_stmt.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_expr_stmt.snow.want => pkg/compiler/parser/testdata/fn_expr_stmt.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_extern.snow => pkg/compiler/parser/testdata/fn_extern.snow +0 -0
R pkg/compiler/testdata/parser/fn_extern.snow.err => pkg/compiler/parser/testdata/fn_extern.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_extern.snow.want => pkg/compiler/parser/testdata/fn_extern.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_mul.snow => pkg/compiler/parser/testdata/fn_mul.snow +0 -0
R pkg/compiler/testdata/parser/fn_mul.snow.err => pkg/compiler/parser/testdata/fn_mul.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_mul.snow.want => pkg/compiler/parser/testdata/fn_mul.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_naked_return.snow => pkg/compiler/parser/testdata/fn_naked_return.snow +0 -0
R pkg/compiler/testdata/parser/fn_naked_return.snow.err => pkg/compiler/parser/testdata/fn_naked_return.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_naked_return.snow.want => pkg/compiler/parser/testdata/fn_naked_return.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_nested_blocks.snow => pkg/compiler/parser/testdata/fn_nested_blocks.snow +0 -0
R pkg/compiler/testdata/parser/fn_nested_blocks.snow.err => pkg/compiler/parser/testdata/fn_nested_blocks.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_nested_blocks.snow.want => pkg/compiler/parser/testdata/fn_nested_blocks.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren.snow => pkg/compiler/parser/testdata/fn_nested_paren.snow +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren.snow.err => pkg/compiler/parser/testdata/fn_nested_paren.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren.snow.want => pkg/compiler/parser/testdata/fn_nested_paren.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren_newline.snow => pkg/compiler/parser/testdata/fn_nested_paren_newline.snow +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren_newline.snow.err => pkg/compiler/parser/testdata/fn_nested_paren_newline.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_nested_paren_newline.snow.want => pkg/compiler/parser/testdata/fn_nested_paren_newline.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_nested_unary.snow => pkg/compiler/parser/testdata/fn_nested_unary.snow +0 -0
R pkg/compiler/testdata/parser/fn_nested_unary.snow.err => pkg/compiler/parser/testdata/fn_nested_unary.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_nested_unary.snow.want => pkg/compiler/parser/testdata/fn_nested_unary.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_paren_expr.snow => pkg/compiler/parser/testdata/fn_paren_expr.snow +0 -0
R pkg/compiler/testdata/parser/fn_paren_expr.snow.err => pkg/compiler/parser/testdata/fn_paren_expr.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_paren_expr.snow.want => pkg/compiler/parser/testdata/fn_paren_expr.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_return_return.snow => pkg/compiler/parser/testdata/fn_return_return.snow +0 -0
R pkg/compiler/testdata/parser/fn_return_return.snow.err => pkg/compiler/parser/testdata/fn_return_return.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_return_return.snow.want => pkg/compiler/parser/testdata/fn_return_return.snow.want +0 -0
R pkg/compiler/testdata/parser/fn_unary.snow => pkg/compiler/parser/testdata/fn_unary.snow +0 -0
R pkg/compiler/testdata/parser/fn_unary.snow.err => pkg/compiler/parser/testdata/fn_unary.snow.err +0 -0
R pkg/compiler/testdata/parser/fn_unary.snow.want => pkg/compiler/parser/testdata/fn_unary.snow.want +0 -0
R pkg/compiler/testdata/parser/let.snow => pkg/compiler/parser/testdata/let.snow +0 -0
R pkg/compiler/testdata/parser/let.snow.err => pkg/compiler/parser/testdata/let.snow.err +0 -0
R pkg/compiler/testdata/parser/let.snow.want => pkg/compiler/parser/testdata/let.snow.want +0 -0
R pkg/compiler/testdata/parser/let_full.snow => pkg/compiler/parser/testdata/let_full.snow +0 -0
R pkg/compiler/testdata/parser/let_full.snow.err => pkg/compiler/parser/testdata/let_full.snow.err +0 -0
R pkg/compiler/testdata/parser/let_full.snow.want => pkg/compiler/parser/testdata/let_full.snow.want +0 -0
R pkg/compiler/testdata/parser/let_infer.snow => pkg/compiler/parser/testdata/let_infer.snow +0 -0
R pkg/compiler/testdata/parser/let_infer.snow.err => pkg/compiler/parser/testdata/let_infer.snow.err +0 -0
R pkg/compiler/testdata/parser/let_infer.snow.want => pkg/compiler/parser/testdata/let_infer.snow.want +0 -0
R pkg/compiler/testdata/parser/let_question.snow => pkg/compiler/parser/testdata/let_question.snow +0 -0
R pkg/compiler/testdata/parser/let_question.snow.err => pkg/compiler/parser/testdata/let_question.snow.err +0 -0
R pkg/compiler/testdata/parser/let_question.snow.want => pkg/compiler/parser/testdata/let_question.snow.want +0 -0
R pkg/compiler/testdata/parser/return.snow => pkg/compiler/parser/testdata/return.snow +0 -0
R pkg/compiler/testdata/parser/return.snow.err => pkg/compiler/parser/testdata/return.snow.err +0 -0
R pkg/compiler/testdata/parser/return.snow.want => pkg/compiler/parser/testdata/return.snow.want +0 -0
R pkg/compiler/testdata/parser/return_return.snow => pkg/compiler/parser/testdata/return_return.snow +0 -0
R pkg/compiler/testdata/parser/return_return.snow.err => pkg/compiler/parser/testdata/return_return.snow.err +0 -0
R pkg/compiler/testdata/parser/return_return.snow.want => pkg/compiler/parser/testdata/return_return.snow.want +0 -0
R pkg/compiler/testdata/parser/var.snow => pkg/compiler/parser/testdata/var.snow +0 -0
R pkg/compiler/testdata/parser/var.snow.err => pkg/compiler/parser/testdata/var.snow.err +0 -0
R pkg/compiler/testdata/parser/var.snow.want => pkg/compiler/parser/testdata/var.snow.want +0 -0
R pkg/compiler/testdata/parser/var_bang.snow => pkg/compiler/parser/testdata/var_bang.snow +0 -0
R pkg/compiler/testdata/parser/var_bang.snow.err => pkg/compiler/parser/testdata/var_bang.snow.err +0 -0
R pkg/compiler/testdata/parser/var_bang.snow.want => pkg/compiler/parser/testdata/var_bang.snow.want +0 -0
R pkg/compiler/testdata/parser/var_comma_for_semi.snow => pkg/compiler/parser/testdata/var_comma_for_semi.snow +0 -0
R pkg/compiler/testdata/parser/var_comma_for_semi.snow.err => pkg/compiler/parser/testdata/var_comma_for_semi.snow.err +0 -0
R pkg/compiler/testdata/parser/var_comma_for_semi.snow.want => pkg/compiler/parser/testdata/var_comma_for_semi.snow.want +0 -0
R pkg/compiler/testdata/parser/var_full.snow => pkg/compiler/parser/testdata/var_full.snow +0 -0
R pkg/compiler/testdata/parser/var_full.snow.err => pkg/compiler/parser/testdata/var_full.snow.err +0 -0
R pkg/compiler/testdata/parser/var_full.snow.want => pkg/compiler/parser/testdata/var_full.snow.want +0 -0
R pkg/compiler/testdata/parser/var_infer.snow => pkg/compiler/parser/testdata/var_infer.snow +0 -0
R pkg/compiler/testdata/parser/var_infer.snow.err => pkg/compiler/parser/testdata/var_infer.snow.err +0 -0
R pkg/compiler/testdata/parser/var_infer.snow.want => pkg/compiler/parser/testdata/var_infer.snow.want +0 -0
R pkg/compiler/testdata/parser/var_invalid_type.snow => pkg/compiler/parser/testdata/var_invalid_type.snow +0 -0
R pkg/compiler/testdata/parser/var_invalid_type.snow.err => pkg/compiler/parser/testdata/var_invalid_type.snow.err +0 -0
R pkg/compiler/testdata/parser/var_invalid_type.snow.want => pkg/compiler/parser/testdata/var_invalid_type.snow.want +0 -0
R pkg/compiler/testdata/parser/var_missing_type_init.snow.notyet => pkg/compiler/parser/testdata/var_missing_type_init.snow.notyet +0 -0
R pkg/compiler/printer.go => pkg/compiler/printer/printer.go +72 -64
@@ 1,43 1,51 @@
package compiler
package printer

import (
	"fmt"
	gotoken "go/token"
	"io"
	"strconv"
	"strings"

	"git.sr.ht/~mna/snow/pkg/compiler/ast"
	"git.sr.ht/~mna/snow/pkg/compiler/token"
)

type posMode int
// PosMode is the mode that controls printing of position information.
type PosMode int

// List of supported position modes.
const (
	posNone    posMode = iota
	posOffsets         // [startoffset:endoffset]
	posLong            // [filename:startline:col:endline:col]
	posRaw             // [%d:%d] of the raw uninterpreted gotoken.Pos values
	PosNone    PosMode = iota
	PosOffsets         // [startoffset:endoffset]
	PosLong            // [filename:startline:col:endline:col]
	PosRaw             // [%d:%d] of the raw uninterpreted gotoken.Pos values
)

type printer struct {
	// options to set before use
	w            io.Writer
	posMode      posMode
	withComments bool
// Printer controls pretty-printing of the AST.
type Printer struct {
	// W is the io.Writer to print to.
	W io.Writer
	// Pos indicates the position mode.
	Pos PosMode
	// WithComments controls printing of comments.
	WithComments bool

	// set during printing
	file   *gotoken.File
	maxPos gotoken.Pos
	file   *token.File
	maxPos token.Pos
	depth  int
	err    error
}

func (p *printer) Print(n node, file *gotoken.File) error {
// Print pretty-prints the AST node n from the specified file.
func (p *Printer) Print(n ast.Node, file *token.File) error {
	p.file = file
	p.maxPos = gotoken.Pos(p.file.Base() + p.file.Size())
	walk(p, n)
	p.maxPos = token.Pos(p.file.Base() + p.file.Size())
	ast.Walk(p, n)
	return p.err
}

func (p *printer) printMsg(s string, indent bool) {
func (p *Printer) printMsg(s string, indent bool) {
	if p.err != nil {
		return
	}


@@ 45,25 53,25 @@ func (p *printer) printMsg(s string, indent bool) {
	if !indent {
		repeat = 0
	}
	_, p.err = fmt.Fprintf(p.w, "%s%s", strings.Repeat("  ", repeat), s)
	_, p.err = fmt.Fprintf(p.W, "%s%s", strings.Repeat("  ", repeat), s)
}

func (p *printer) clamp(pos gotoken.Pos) gotoken.Pos {
func (p *Printer) clamp(pos token.Pos) token.Pos {
	if pos > p.maxPos {
		return p.maxPos
	}
	return pos
}

func (p *printer) printPosln(n node) {
func (p *Printer) printPosln(n ast.Node) {
	if p.err != nil {
		return
	}

	var startLbl, endLbl string
	start, end := n.Pos(), n.End()
	switch p.posMode {
	case posOffsets:
	switch p.Pos {
	case PosOffsets:
		startLbl, endLbl = "-", "-"
		if start.IsValid() {
			startLbl = strconv.Itoa(p.file.Offset(start))


@@ 72,7 80,7 @@ func (p *printer) printPosln(n node) {
			endLbl = strconv.Itoa(p.file.Offset(p.clamp(end)))
		}

	case posLong:
	case PosLong:
		if start.IsValid() {
			lstart := p.file.Position(start)
			startLbl = fmt.Sprintf("%s:%d:%d", lstart.Filename, lstart.Line, lstart.Column)


@@ 86,14 94,14 @@ func (p *printer) printPosln(n node) {
			endLbl = ":-:-"
		}

	case posRaw:
	case PosRaw:
		startLbl, endLbl = strconv.Itoa(int(start)), strconv.Itoa(int(end))
	}

	_, p.err = fmt.Fprintf(p.w, " [%s:%s]\n", startLbl, endLbl)
	_, p.err = fmt.Fprintf(p.W, " [%s:%s]\n", startLbl, endLbl)
}

func (p *printer) Visit(n node) visitor {
func (p *Printer) Visit(n ast.Node) ast.Visitor {
	if n == nil || p.err != nil {
		p.depth--
		return nil


@@ 101,69 109,69 @@ func (p *printer) Visit(n node) visitor {
	p.depth++

	switch n := n.(type) {
	case *file:
		p.printMsg(fmt.Sprintf("file [%d, #%d]", len(n.stmts), len(n.comments)), true)
	case *fnDef:
	case *ast.File:
		p.printMsg(fmt.Sprintf("file [%d, #%d]", len(n.Stmts), len(n.Comments)), true)
	case *ast.FnDef:
		p.printMsg("fn", true)
	case *fnSig:
	case *ast.FnSig:
		ret := 0
		if n.retTyp != nil {
		if n.RetType != nil {
			ret = 1
		}
		p.printMsg(fmt.Sprintf("sig [%d->%d]", len(n.params), ret), true)
	case *paramDef:
		p.printMsg(fmt.Sprintf("sig [%d->%d]", len(n.Params), ret), true)
	case *ast.ParamDef:
		p.printMsg("param", true)
	case *block:
		p.printMsg(fmt.Sprintf("block [%d]", len(n.stmts)), true)
	case *attr:
		p.printMsg(fmt.Sprintf("@ [%d]", len(n.fields)), true)
	case *keyValue:
	case *ast.Block:
		p.printMsg(fmt.Sprintf("block [%d]", len(n.Stmts)), true)
	case *ast.Attr:
		p.printMsg(fmt.Sprintf("@ [%d]", len(n.Fields)), true)
	case *ast.KeyValue:
		p.printMsg("field", true)
	case *varDef:
		lbl := n.kw.String()
		if n.colon.IsValid() {
	case *ast.VarDef:
		lbl := n.Kw.String()
		if n.Colon.IsValid() {
			lbl += ":"
		}
		if n.assign.IsValid() {
		if n.Assign.IsValid() {
			lbl += "="
		}
		p.printMsg(lbl, true)
	case *returnStmt:
	case *ast.ReturnStmt:
		p.printMsg("return", true)
	case *binaryExpr:
		p.printMsg(fmt.Sprintf("binary [%s]", n.op), true)
	case *unaryExpr:
		p.printMsg(fmt.Sprintf("unary [%s]", n.op), true)
	case *parenExpr:
	case *ast.BinaryExpr:
		p.printMsg(fmt.Sprintf("binary [%s]", n.Op), true)
	case *ast.UnaryExpr:
		p.printMsg(fmt.Sprintf("unary [%s]", n.Op), true)
	case *ast.ParenExpr:
		p.printMsg("paren", true)
	case *assignStmt:
	case *ast.AssignStmt:
		p.printMsg("assign", true)
	case *exprStmt:
	case *ast.ExprStmt:
		p.printMsg("expr", true)
	case *basicLit:
		p.printMsg(fmt.Sprintf("%s [%s]", n.literal, n.value), true)
	case *ident:
		p.printMsg(fmt.Sprintf("ident [%s]", n.name), true)
	case *badStmt:
	case *ast.BasicLit:
		p.printMsg(fmt.Sprintf("%s [%s]", n.Literal, n.Value), true)
	case *ast.Ident:
		p.printMsg(fmt.Sprintf("ident [%s]", n.Name), true)
	case *ast.BadStmt:
		p.printMsg("bad stmt", true)
	case *badExpr:
	case *ast.BadExpr:
		p.printMsg("bad expr", true)
	case *commentGroup:
		if !p.withComments {
	case *ast.CommentGroup:
		if !p.WithComments {
			return nil
		}
		p.printMsg(fmt.Sprintf("comments [%d]", len(n.list)), true)
	case *comment:
		if !p.withComments {
		p.printMsg(fmt.Sprintf("comments [%d]", len(n.List)), true)
	case *ast.Comment:
		if !p.WithComments {
			return nil
		}
		p.printMsg(fmt.Sprintf("comment [%s]", n.text), true)
		p.printMsg(fmt.Sprintf("comment [%s]", n.Text), true)

	default:
		panic(fmt.Sprintf("printer: unexpected node type %T", n))
	}

	if p.posMode != posNone {
	if p.Pos != PosNone {
		p.printPosln(n)
	} else {
		p.printMsg("\n", false)

A pkg/compiler/printer/printer_test.go => pkg/compiler/printer/printer_test.go +234 -0
@@ 0,0 1,234 @@
package printer

import (
	"bytes"
	"fmt"
	"strings"
	"testing"

	"git.sr.ht/~mna/snow/pkg/compiler/ast"
	"git.sr.ht/~mna/snow/pkg/compiler/token"
	"github.com/kylelemons/godebug/diff"
)

func TestPrinter(t *testing.T) {
	fs := token.NewFileSet()
	fsf := fs.AddFile("test", -1, 100)

	f := &ast.File{
		Comments: []*ast.CommentGroup{
			{
				List: []*ast.Comment{
					{Pound: 1, Text: "#abc"},
					{Pound: 5, Text: "#def"},
				},
			},
		},
		Stmts: []ast.Stmt{
			&ast.VarDef{
				Kw:    token.Var,
				KwPos: 9,
				Name: &ast.Ident{
					Name:     "a",
					IdentPos: 13,
				},
				Colon: 14,
				Type: &ast.Ident{
					Name:     "int",
					IdentPos: 16,
				},
				Assign: 20,
				Value: &ast.BasicLit{
					LitPos:  22,
					Literal: token.Int,
					Value:   "1",
				},
			},

			&ast.FnDef{
				Fn: 24,
				Name: &ast.Ident{
					Name:     "add",
					IdentPos: 27,
				},
				Signature: &ast.FnSig{
					Lparen: 31,
					Params: []*ast.ParamDef{
						{
							Name: &ast.Ident{
								Name:     "x",
								IdentPos: 32,
							},
							Colon: 33,
							Type: &ast.Ident{
								Name:     "int",
								IdentPos: 35,
							},
							Comma: 38,
						},
						{
							Name: &ast.Ident{
								Name:     "y",
								IdentPos: 40,
							},
							Colon: 41,
							Type: &ast.Ident{
								Name:     "int",
								IdentPos: 43,
							},
						},
					},
					Rparen: 44,
					Rarrow: 46,
					RetType: &ast.Ident{
						Name:     "int",
						IdentPos: 49,
					},
				},
				Body: &ast.Block{
					Lbrace: 53,
					Stmts: []ast.Stmt{
						&ast.ReturnStmt{
							Ret: 57,
							Value: &ast.BinaryExpr{
								Left: &ast.Ident{
									Name:     "x",
									IdentPos: 65,
								},
								Op:    token.Add,
								OpPos: 66,
								Right: &ast.Ident{
									Name:     "y",
									IdentPos: 67,
								},
							},
						},
					},
					Rbrace: 69,
				},
			},
		},
	}

	wantNoPos := `
file [2, #1]
    var:=
      ident [a]
      ident [int]
      int [1]
    fn
      ident [add]
      sig [2->1]
        param
          ident [x]
          ident [int]
        param
          ident [y]
          ident [int]
        ident [int]
      block [1]
        return
          binary [+]
            ident [x]
            ident [y]
`
	wantComments := `
file [2, #1]
  comments [2]
    comment [#abc]
    comment [#def]
  var:=
    ident [a]
    ident [int]
    int [1]
  fn
    ident [add]
    sig [2->1]
      param
        ident [x]
        ident [int]
      param
        ident [y]
        ident [int]
      ident [int]
    block [1]
      return
        binary [+]
          ident [x]
          ident [y]
`
	wantShort := `
file [2, #1] [0:69]
    var:= [8:22]
      ident [a] [12:13]
      ident [int] [15:18]
      int [1] [21:22]
    fn [23:69]
      ident [add] [26:29]
      sig [2->1] [30:51]
        param [31:38]
          ident [x] [31:32]
          ident [int] [34:37]
        param [39:45]
          ident [y] [39:40]
          ident [int] [42:45]
        ident [int] [48:51]
      block [1] [52:69]
        return [56:67]
          binary [+] [64:67]
            ident [x] [64:65]
            ident [y] [66:67]
`
	wantLong := `
file [2, #1] [test:1:1::1:70]
    var:= [test:1:9::1:23]
      ident [a] [test:1:13::1:14]
      ident [int] [test:1:16::1:19]
      int [1] [test:1:22::1:23]
    fn [test:1:24::1:70]
      ident [add] [test:1:27::1:30]
      sig [2->1] [test:1:31::1:52]
        param [test:1:32::1:39]
          ident [x] [test:1:32::1:33]
          ident [int] [test:1:35::1:38]
        param [test:1:40::1:46]
          ident [y] [test:1:40::1:41]
          ident [int] [test:1:43::1:46]
        ident [int] [test:1:49::1:52]
      block [1] [test:1:53::1:70]
        return [test:1:57::1:68]
          binary [+] [test:1:65::1:68]
            ident [x] [test:1:65::1:66]
            ident [y] [test:1:67::1:68]
`

	cases := []struct {
		posMode      PosMode
		withComments bool
		want         string
	}{
		{PosNone, false, wantNoPos},
		{PosOffsets, false, wantShort},
		{PosLong, false, wantLong},
		{PosNone, true, wantComments},
	}
	for _, c := range cases {
		t.Run(fmt.Sprintf("%v %v", c.posMode, c.withComments), func(t *testing.T) {
			var buf bytes.Buffer
			p := Printer{
				W:            &buf,
				WithComments: c.withComments,
				Pos:          c.posMode,
			}
			if err := p.Print(f, fsf); err != nil {
				t.Fatal(err)
			}

			want := strings.TrimSpace(c.want)
			got := strings.TrimSpace(buf.String())
			if patch := diff.Diff(got, want); patch != "" {
				t.Fatalf("diff:\n%s\n", patch)
			}
		})
	}
}

D pkg/compiler/printer_test.go => pkg/compiler/printer_test.go +0 -233
@@ 1,233 0,0 @@
package compiler

import (
	"bytes"
	"fmt"
	gotoken "go/token"
	"strings"
	"testing"

	"github.com/kylelemons/godebug/diff"
)

func TestPrinter(t *testing.T) {
	fs := gotoken.NewFileSet()
	fsf := fs.AddFile("test", -1, 100)

	f := &file{
		comments: []*commentGroup{
			{
				list: []*comment{
					{pound: 1, text: "#abc"},
					{pound: 5, text: "#def"},
				},
			},
		},
		stmts: []stmt{
			&varDef{
				kw:    Var,
				kwPos: 9,
				name: &ident{
					name: "a",
					pos:  13,
				},
				colon: 14,
				typ: &ident{
					name: "int",
					pos:  16,
				},
				assign: 20,
				value: &basicLit{
					pos:     22,
					literal: Int,
					value:   "1",
				},
			},

			&fnDef{
				fn: 24,
				name: &ident{
					name: "add",
					pos:  27,
				},
				signature: &fnSig{
					lparen: 31,
					params: []*paramDef{
						{
							name: &ident{
								name: "x",
								pos:  32,
							},
							colon: 33,
							typ: &ident{
								name: "int",
								pos:  35,
							},
							comma: 38,
						},
						{
							name: &ident{
								name: "y",
								pos:  40,
							},
							colon: 41,
							typ: &ident{
								name: "int",
								pos:  43,
							},
						},
					},
					rparen: 44,
					rarrow: 46,
					retTyp: &ident{
						name: "int",
						pos:  49,
					},
				},
				body: &block{
					lbrace: 53,
					stmts: []stmt{
						&returnStmt{
							ret: 57,
							value: &binaryExpr{
								left: &ident{
									name: "x",
									pos:  65,
								},
								op:    Add,
								opPos: 66,
								right: &ident{
									name: "y",
									pos:  67,
								},
							},
						},
					},
					rbrace: 69,
				},
			},
		},
	}

	wantNoPos := `
file [2, #1]
    var:=
      ident [a]
      ident [int]
      int [1]
    fn
      ident [add]
      sig [2->1]
        param
          ident [x]
          ident [int]
        param
          ident [y]
          ident [int]
        ident [int]
      block [1]
        return
          binary [+]
            ident [x]
            ident [y]
`
	wantComments := `
file [2, #1]
  comments [2]
    comment [#abc]
    comment [#def]
  var:=
    ident [a]
    ident [int]
    int [1]
  fn
    ident [add]
    sig [2->1]
      param
        ident [x]
        ident [int]
      param
        ident [y]
        ident [int]
      ident [int]
    block [1]
      return
        binary [+]
          ident [x]
          ident [y]
`
	wantShort := `
file [2, #1] [0:69]
    var:= [8:22]
      ident [a] [12:13]
      ident [int] [15:18]
      int [1] [21:22]
    fn [23:69]
      ident [add] [26:29]
      sig [2->1] [30:51]
        param [31:38]
          ident [x] [31:32]
          ident [int] [34:37]
        param [39:45]
          ident [y] [39:40]
          ident [int] [42:45]
        ident [int] [48:51]
      block [1] [52:69]
        return [56:67]
          binary [+] [64:67]
            ident [x] [64:65]
            ident [y] [66:67]
`
	wantLong := `
file [2, #1] [test:1:1::1:70]
    var:= [test:1:9::1:23]
      ident [a] [test:1:13::1:14]
      ident [int] [test:1:16::1:19]
      int [1] [test:1:22::1:23]
    fn [test:1:24::1:70]
      ident [add] [test:1:27::1:30]
      sig [2->1] [test:1:31::1:52]
        param [test:1:32::1:39]
          ident [x] [test:1:32::1:33]
          ident [int] [test:1:35::1:38]
        param [test:1:40::1:46]
          ident [y] [test:1:40::1:41]
          ident [int] [test:1:43::1:46]
        ident [int] [test:1:49::1:52]
      block [1] [test:1:53::1:70]
        return [test:1:57::1:68]
          binary [+] [test:1:65::1:68]
            ident [x] [test:1:65::1:66]
            ident [y] [test:1:67::1:68]
`

	cases := []struct {
		posMode      posMode
		withComments bool
		want         string
	}{
		{posNone, false, wantNoPos},
		{posOffsets, false, wantShort},
		{posLong, false, wantLong},
		{posNone, true, wantComments},
	}
	for _, c := range cases {
		t.Run(fmt.Sprintf("%v %v", c.posMode, c.withComments), func(t *testing.T) {
			var buf bytes.Buffer
			p := printer{
				w:            &buf,
				withComments: c.withComments,
				posMode:      c.posMode,
			}
			if err := p.Print(f, fsf); err != nil {
				t.Fatal(err)
			}

			want := strings.TrimSpace(c.want)
			got := strings.TrimSpace(buf.String())
			if patch := diff.Diff(got, want); patch != "" {
				t.Fatalf("diff:\n%s\n", patch)
			}
		})
	}
}

R pkg/compiler/scanner.go => pkg/compiler/scanner/scanner.go +42 -36
@@ 1,24 1,29 @@
package compiler
package scanner

import (
	"fmt"
	gotoken "go/token"
	"go/scanner"
	"path/filepath"
	"unicode"
	"unicode/utf8"

	"git.sr.ht/~mna/snow/pkg/compiler/token"
)

type ErrorList = scanner.ErrorList

// byte order mark, only permitted as very first character
const bom = 0xFEFF

type scanner struct {
// Scanner tokenizes source files for the parser to consume.
type Scanner struct {
	// immutable state
	file *gotoken.File // source file handle
	dir  string        // directory portion of file.Name()
	src  []byte        // source
	file *token.File // source file handle
	dir  string      // directory portion of file.Name()
	src  []byte      // source

	// setup an error handler to receive scanning errors
	err func(pos gotoken.Position, msg string)
	err func(pos token.Position, msg string)

	// mutable scanning state
	cur  rune // current character


@@ 31,7 36,8 @@ type scanner struct {
	errCount int
}

func (s *scanner) init(file *gotoken.File, src []byte, errHandler func(gotoken.Position, string)) {
// Init initializes the scanner to tokenize a new file.
func (s *Scanner) Init(file *token.File, src []byte, errHandler func(token.Position, string)) {
	if file.Size() != len(src) {
		panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
	}


@@ 55,7 61,7 @@ func (s *scanner) init(file *gotoken.File, src []byte, errHandler func(gotoken.P
}

// read the next Unicode char into s.cur; s.cur < 0 means end-of-file.
func (s *scanner) next() {
func (s *Scanner) next() {
	if s.roff >= len(s.src) {
		s.off = len(s.src)
		if s.cur == '\n' {


@@ 91,14 97,14 @@ func (s *scanner) next() {

// peek returns the byte following the most recently read character without
// advancing the scanner. If the scanner is at EOF, peek returns 0.
func (s *scanner) peek() byte {
func (s *Scanner) peek() byte {
	if s.roff < len(s.src) {
		return s.src[s.roff]
	}
	return 0
}

// scan returns the next token in the source file. It automatically inserts
// Scan returns the next token in the source file. It automatically inserts
// semicolons following those rules:
//
// - if the non-comment final token of a line is one of:


@@ 117,7 123,7 @@ func (s *scanner) peek() byte {
// s.insertSemi is set to true. If on the next call to scan a newline or a
// comment is encountered, the semicolon token is returned and s.insertSemi
// is reset to false.
func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
	s.skipWhitespace()

	// current token start


@@ 129,12 135,12 @@ func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
	switch cur := s.cur; {
	case isLetter(cur):
		lit = s.ident()
		tok = Ident
		tok = token.Ident
		if len(lit) > 1 {
			// keywords are longer than one letter - avoid lookup otherwise
			tok = lookupKw(lit)
			tok = token.LookupKw(lit)
		}
		insertSemi = (tok == Ident || tok == Return)
		insertSemi = (tok == token.Ident || tok == token.Return)

	case isDecimal(cur) || cur == '.' && isDecimal(rune(s.peek())):
		tok, lit = s.number()


@@ 152,13 158,13 @@ func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
		case -1:
			if s.insertSemi {
				s.insertSemi = false
				return pos, Semicolon, ""
				return pos, token.Semicolon, ""
			}
			tok = EOF
			tok = token.EOF

		case '\n':
			s.insertSemi = false
			return pos, Semicolon, "\n"
			return pos, token.Semicolon, "\n"

		case '#':
			if s.insertSemi {


@@ 166,30 172,30 @@ func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
				s.cur = '#'
				s.off = s.file.Offset(pos)
				s.roff = roff
				return pos, Semicolon, "#"
				return pos, token.Semicolon, "#"
			}
			tok = Comment
			tok = token.Comment
			lit = s.comment()

		case '"':
			tok = String
			tok = token.String
			lit = s.string()
			insertSemi = true

		case ':', ';', ',', '(', ')', '{', '}', '+', '*', '/', '%', '=', '@':
			tok = lookupOp(string(cur))
			insertSemi = (tok == Rparen || tok == Rbrace)
			tok = token.LookupOp(string(cur))
			insertSemi = (tok == token.Rparen || tok == token.Rbrace)
			// special-case for semicolons, set the literal to be able to tell apart the
			// explicit vs implicit semicolons.
			if tok == Semicolon {
			if tok == token.Semicolon {
				lit = ";"
			}

		case '-':
			tok = Sub
			tok = token.Sub
			if s.cur == '>' {
				s.next()
				tok = Rarrow
				tok = token.Rarrow
			}

		default:


@@ 197,7 203,7 @@ func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
			if cur != bom && cur != 0 {
				s.errorf(s.file.Offset(pos), "illegal character %#U", cur)
			}
			tok = Illegal
			tok = token.Illegal
			lit = string(cur)
			// keep current insertSemi state
			insertSemi = s.insertSemi


@@ 208,7 214,7 @@ func (s *scanner) scan() (pos gotoken.Pos, tok token, lit string) {
	return pos, tok, lit
}

func (s *scanner) ident() string {
func (s *Scanner) ident() string {
	off := s.off
	for isLetter(s.cur) || isDigit(s.cur) {
		s.next()


@@ 220,16 226,16 @@ func (s *scanner) ident() string {
	return string(s.src[off:s.off])
}

func (s *scanner) number() (token, string) {
func (s *Scanner) number() (token.Token, string) {
	off := s.off
	for isDigit(s.cur) {
		s.next()
	}
	// TODO: for now, only integers
	return Int, string(s.src[off:s.off])
	return token.Int, string(s.src[off:s.off])
}

func (s *scanner) comment() string {
func (s *Scanner) comment() string {
	// '#' opening already consumed
	off := s.off - 1
	for s.cur != '\n' && s.cur != -1 {


@@ 238,7 244,7 @@ func (s *scanner) comment() string {
	return string(s.src[off:s.off])
}

func (s *scanner) string() string {
func (s *Scanner) string() string {
	// '"' opening already consumed
	off := s.off - 1



@@ 264,7 270,7 @@ func (s *scanner) string() string {
// escaped quote. In case of a syntax error, it stops at the offending
// character (without consuming it) and returns false. Otherwise
// it returns true.
func (s *scanner) escape(quote rune) bool {
func (s *Scanner) escape(quote rune) bool {
	off := s.off

	var n int


@@ 323,18 329,18 @@ func (s *scanner) escape(quote rune) bool {
	return true
}

func (s *scanner) error(off int, msg string) {
func (s *Scanner) error(off int, msg string) {
	if s.err != nil {
		s.err(s.file.Position(s.file.Pos(off)), msg)
	}
	s.errCount++
}

func (s *scanner) errorf(off int, format string, args ...interface{}) {
func (s *Scanner) errorf(off int, format string, args ...interface{}) {
	s.error(off, fmt.Sprintf(format, args...))
}

func (s *scanner) skipWhitespace() {
func (s *Scanner) skipWhitespace() {
	for s.cur == ' ' || s.cur == '\t' || s.cur == '\n' && !s.insertSemi || s.cur == '\r' {
		s.next()
	}

R pkg/compiler/scanner_test.go => pkg/compiler/scanner/scanner_test.go +85 -84
@@ 1,14 1,15 @@
package compiler
package scanner

import (
	"fmt"
	gotoken "go/token"
	"testing"
	"unicode/utf8"

	"git.sr.ht/~mna/snow/pkg/compiler/token"
)

type tokLit struct {
	tok token
	tok token.Token
	lit string
}



@@ 22,215 23,215 @@ func TestScanner(t *testing.T) {
		{
			"#",
			[]tokLit{
				{Comment, "#"},
				{token.Comment, "#"},
			},
			0,
		},
		{
			"a",
			[]tokLit{
				{Ident, "a"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			"return a",
			[]tokLit{
				{Return, "return"},
				{Ident, "a"},
				{Semicolon, ""},
				{token.Return, "return"},
				{token.Ident, "a"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			"let x: int = 123",
			[]tokLit{
				{Let, "let"},
				{Ident, "x"},
				{Colon, ""},
				{Ident, "int"},
				{Assign, ""},
				{Int, "123"},
				{Semicolon, ""},
				{token.Let, "let"},
				{token.Ident, "x"},
				{token.Colon, ""},
				{token.Ident, "int"},
				{token.Assign, ""},
				{token.Int, "123"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			`var x = "a\bc\001\xaF\u1234\U00012233"`,
			[]tokLit{
				{Var, "var"},
				{Ident, "x"},
				{Assign, ""},
				{String, `"a\bc\001\xaF\u1234\U00012233"`},
				{Semicolon, ""},
				{token.Var, "var"},
				{token.Ident, "x"},
				{token.Assign, ""},
				{token.String, `"a\bc\001\xaF\u1234\U00012233"`},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			`var x = y - 1`,
			[]tokLit{
				{Var, "var"},
				{Ident, "x"},
				{Assign, ""},
				{Ident, "y"},
				{Sub, ""},
				{Int, "1"},
				{Semicolon, ""},
				{token.Var, "var"},
				{token.Ident, "x"},
				{token.Assign, ""},
				{token.Ident, "y"},
				{token.Sub, ""},
				{token.Int, "1"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			`fn() -> int { }`,
			[]tokLit{
				{Fn, "fn"},
				{Lparen, ""},
				{Rparen, ""},
				{Rarrow, ""},
				{Ident, "int"},
				{Lbrace, ""},
				{Rbrace, ""},
				{Semicolon, ""},
				{token.Fn, "fn"},
				{token.Lparen, ""},
				{token.Rparen, ""},
				{token.Rarrow, ""},
				{token.Ident, "int"},
				{token.Lbrace, ""},
				{token.Rbrace, ""},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			string(bom) + "a",
			[]tokLit{
				{Ident, "a"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			string(bom) + "a#b\nc",
			[]tokLit{
				{Ident, "a"},
				{Semicolon, "#"},
				{Comment, "#b"},
				{Ident, "c"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Semicolon, "#"},
				{token.Comment, "#b"},
				{token.Ident, "c"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			"a" + string(bom) + "b",
			[]tokLit{
				{Ident, "a"},
				{Illegal, string(bom)},
				{Ident, "b"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Illegal, string(bom)},
				{token.Ident, "b"},
				{token.Semicolon, ""},
			},
			1,
		},
		{
			"a\x00b",
			[]tokLit{
				{Ident, "a"},
				{Illegal, "\x00"},
				{Ident, "b"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Illegal, "\x00"},
				{token.Ident, "b"},
				{token.Semicolon, ""},
			},
			1,
		},
		{
			"a\xffb",
			[]tokLit{
				{Ident, "a"},
				{Illegal, string(utf8.RuneError)},
				{Ident, "b"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Illegal, string(utf8.RuneError)},
				{token.Ident, "b"},
				{token.Semicolon, ""},
			},
			2, // illegal encoding + illegal char RuneError
		},
		{
			"a$b",
			[]tokLit{
				{Ident, "a"},
				{Illegal, "$"},
				{Ident, "b"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Illegal, "$"},
				{token.Ident, "b"},
				{token.Semicolon, ""},
			},
			1,
		},
		{
			"\"ab\n",
			[]tokLit{
				{String, "\"ab"},
				{Semicolon, "\n"},
				{token.String, "\"ab"},
				{token.Semicolon, "\n"},
			},
			1,
		},
		{
			`"ab\`,
			[]tokLit{
				{String, `"ab\`},
				{Semicolon, ""},
				{token.String, `"ab\`},
				{token.Semicolon, ""},
			},
			2, // escape not terminated + string not terminated
		},
		{
			`"ab\xaG"`,
			[]tokLit{
				{String, `"ab\xaG"`},
				{Semicolon, ""},
				{token.String, `"ab\xaG"`},
				{token.Semicolon, ""},
			},
			1,
		},
		{
			`"ab\xa`,
			[]tokLit{
				{String, `"ab\xa`},
				{Semicolon, ""},
				{token.String, `"ab\xa`},
				{token.Semicolon, ""},
			},
			2, // escape + string not terminated
		},
		{
			`"\U99999999"`,
			[]tokLit{
				{String, `"\U99999999"`},
				{Semicolon, ""},
				{token.String, `"\U99999999"`},
				{token.Semicolon, ""},
			},
			1,
		},
		{
			"a;b\nc",
			[]tokLit{
				{Ident, "a"},
				{Semicolon, ";"},
				{Ident, "b"},
				{Semicolon, "\n"},
				{Ident, "c"},
				{Semicolon, ""},
				{token.Ident, "a"},
				{token.Semicolon, ";"},
				{token.Ident, "b"},
				{token.Semicolon, "\n"},
				{token.Ident, "c"},
				{token.Semicolon, ""},
			},
			0,
		},
		{
			"a#",
			[]tokLit{
				{Ident, "a"},
				{Semicolon, "#"},
				{Comment, "#"},
				{token.Ident, "a"},
				{token.Semicolon, "#"},
				{token.Comment, "#"},
			},
			0,
		},
	}
	for i, c := range cases {
		t.Run(fmt.Sprintf("%d: %s", i, c.in), func(t *testing.T) {
			fset := gotoken.NewFileSet()
			fset := token.NewFileSet()
			file := fset.AddFile("test", -1, len(c.in))

			var s scanner
			eh := func(pos gotoken.Position, msg string) {
			var s Scanner
			eh := func(pos token.Position, msg string) {
				t.Logf("%v: %s", pos, msg)
			}

			s.init(file, []byte(c.in), eh)
			s.Init(file, []byte(c.in), eh)

			n := 0
			for {
				_, tok, lit := s.scan()
				if tok == EOF {
				_, tok, lit := s.Scan()
				if tok == token.EOF {
					break
				}


R pkg/compiler/testdata/codegen/fn_add.snow => pkg/compiler/testdata/fn_add.snow +0 -0
R pkg/compiler/token.go => pkg/compiler/token/token.go +40 -25
@@ 1,12 1,25 @@
package compiler
package token

import "go/token"

type (
	File     = token.File
	FileSet  = token.FileSet
	Position = token.Position
	Pos      = token.Pos
)

const NoPos = token.NoPos

var NewFileSet = token.NewFileSet

// the set of lexical tokens of the programming language.
type token int
type Token int

// The list of tokens.
const (
	// Special tokens
	Illegal token = iota
	Illegal Token = iota
	EOF
	Comment



@@ 79,37 92,39 @@ var tokens = [...]string{
	Return: "return",
}

func (tok token) String() string {
func (tok Token) String() string {
	return tokens[tok]
}

var (
	keywords  map[string]token
	operators map[string]token
	keywords = func() map[string]Token {
		kw := make(map[string]Token)
		for i := kwStart + 1; i < kwEnd; i++ {
			kw[tokens[i]] = i
		}
		return kw
	}()
	operators = func() map[string]Token {
		ops := make(map[string]Token)
		for i := opStart + 1; i < opEnd; i++ {
			ops[tokens[i]] = i
		}
		return ops
	}()
)

func init() {
	keywords = make(map[string]token)
	for i := kwStart + 1; i < kwEnd; i++ {
		keywords[tokens[i]] = i
	}

	operators = make(map[string]token)
	for i := opStart + 1; i < opEnd; i++ {
		operators[tokens[i]] = i
	}
}

// maps an identifier to its keyword token or Ident (if not a keyword).
func lookupKw(ident string) token {
// LookupKw maps an identifier to its keyword token or Ident (if not a
// keyword).
func LookupKw(ident string) Token {
	if tok, ok := keywords[ident]; ok {
		return tok
	}
	return Ident
}

// maps an operator to its operator token or Illegal (if not an operator).
func lookupOp(op string) token {
// LookupOp maps an operator to its operator token or Illegal (if not an
// operator).
func LookupOp(op string) Token {
	if tok, ok := operators[op]; ok {
		return tok
	}


@@ 118,12 133,12 @@ func lookupOp(op string) token {

// Literal returns true for tokens corresponding to identifiers
// and basic type literals, false otherwise.
func (tok token) literal() bool { return litStart < tok && tok < litEnd }
func (tok Token) Literal() bool { return litStart < tok && tok < litEnd }

// Operator returns true for tokens corresponding to operators and
// delimiters, false otherwise.
func (tok token) operator() bool { return opStart < tok && tok < opEnd }
func (tok Token) Operator() bool { return opStart < tok && tok < opEnd }

// Keyword returns true for tokens corresponding to keywords,
// false otherwise.
func (tok token) keyword() bool { return kwStart < tok && tok < kwEnd }
func (tok Token) Keyword() bool { return kwStart < tok && tok < kwEnd }

R pkg/compiler/token_test.go => pkg/compiler/token/token_test.go +15 -15
@@ 1,9 1,9 @@
package compiler
package token

import "testing"

func TestToken(t *testing.T) {
	ignoreTokens := map[token]bool{
	ignoreTokens := map[Token]bool{
		litStart: true,
		litEnd:   true,
		opStart:  true,


@@ 24,60 24,60 @@ func TestToken(t *testing.T) {
		}
	})

	t.Run("lookupKw", func(t *testing.T) {
	t.Run("LookupKw", func(t *testing.T) {
		for i := kwStart + 1; i < kwEnd; i++ {
			if tok := lookupKw(i.String()); !tok.keyword() {
			if tok := LookupKw(i.String()); !tok.Keyword() {
				t.Fatalf("token %s does not return a keyword token", i)
			}
		}

		for i := opStart + 1; i < opEnd; i++ {
			if tok := lookupKw(i.String()); tok.keyword() {
			if tok := LookupKw(i.String()); tok.Keyword() {
				t.Fatalf("token %s should not return a keyword token", tok)
			}
		}

		ids := []string{"a", "notAKeyword"}
		for _, id := range ids {
			if tok := lookupKw(id); tok != Ident {
			if tok := LookupKw(id); tok != Ident {
				t.Fatalf("identifier %s does not return an Ident token, returns: %s", id, tok)
			}
		}
	})

	t.Run("lookupOp", func(t *testing.T) {
	t.Run("LookupOp", func(t *testing.T) {
		for i := opStart + 1; i < opEnd; i++ {
			if tok := lookupOp(i.String()); !tok.operator() {
			if tok := LookupOp(i.String()); !tok.Operator() {
				t.Fatalf("token %s does not return an operator token", i)
			}
		}

		for i := kwStart + 1; i < kwEnd; i++ {
			if tok := lookupOp(i.String()); tok != Illegal {
			if tok := LookupOp(i.String()); tok != Illegal {
				t.Fatalf("token %s does not return an Illegal token, returns: %s", i, tok)
			}
		}
	})

	t.Run("literal", func(t *testing.T) {
	t.Run("Literal", func(t *testing.T) {
		for i := litStart + 1; i < litEnd; i++ {
			if !i.literal() {
			if !i.Literal() {
				t.Fatalf("token %s should be a literal", i)
			}
		}
	})

	t.Run("operator", func(t *testing.T) {
	t.Run("Operator", func(t *testing.T) {
		for i := opStart + 1; i < opEnd; i++ {
			if !i.operator() {
			if !i.Operator() {
				t.Fatalf("token %s should be an operator", i)
			}
		}
	})

	t.Run("keyword", func(t *testing.T) {
	t.Run("Keyword", func(t *testing.T) {
		for i := kwStart + 1; i < kwEnd; i++ {
			if !i.keyword() {
			if !i.Keyword() {
				t.Fatalf("token %s should be a keyword", i)
			}
		}

D pkg/compiler/visitor.go => pkg/compiler/visitor.go +0 -133
@@ 1,133 0,0 @@
package compiler

import "fmt"

type visitor interface {
	Visit(n node) (w visitor)
}

func walk(v visitor, node node) {
	if v = v.Visit(node); v == nil {
		return
	}

	switch n := node.(type) {
	case *file:
		for _, cg := range n.comments {
			walk(v, cg)
		}
		for _, stmt := range n.stmts {
			walk(v, stmt)
		}

	case *fnDef:
		for _, attr := range n.attrs {
			walk(v, attr)
		}
		if n.name != nil {
			walk(v, n.name)
		}
		if n.signature != nil {
			walk(v, n.signature)
		}
		if n.body != nil {
			walk(v, n.body)
		}

	case *fnSig:
		for _, param := range n.params {
			walk(v, param)
		}
		if n.retTyp != nil {
			walk(v, n.retTyp)
		}

	case *paramDef:
		if n.name != nil {
			walk(v, n.name)
		}
		if n.typ != nil {
			walk(v, n.typ)
		}

	case *block:
		for _, stmt := range n.stmts {
			walk(v, stmt)
		}

	case *attr:
		if n.name != nil {
			walk(v, n.name)
		}
		for _, field := range n.fields {
			walk(v, field)
		}

	case *keyValue:
		if n.key != nil {
			walk(v, n.key)
		}
		if n.value != nil {
			walk(v, n.value)
		}

	case *varDef:
		if n.name != nil {
			walk(v, n.name)
		}
		if n.typ != nil {
			walk(v, n.typ)
		}
		if n.value != nil {
			walk(v, n.value)
		}

	case *returnStmt:
		if n.value != nil {
			walk(v, n.value)
		}

	case *binaryExpr:
		if n.left != nil {
			walk(v, n.left)
		}
		if n.right != nil {
			walk(v, n.right)
		}

	case *unaryExpr:
		if n.right != nil {
			walk(v, n.right)
		}

	case *parenExpr:
		if n.value != nil {
			walk(v, n.value)
		}

	case *assignStmt:
		if n.left != nil {
			walk(v, n.left)
		}
		if n.right != nil {
			walk(v, n.right)
		}

	case *exprStmt:
		if n.value != nil {
			walk(v, n.value)
		}

	case *commentGroup:
		for _, comment := range n.list {
			walk(v, comment)
		}

	case *basicLit, *ident, *comment, *badStmt, *badExpr:
		// no child to step into

	default:
		panic(fmt.Sprintf("walk: unexpected node type %T", n))
	}
	v.Visit(nil)
}