~mna/snow

c0bc63be93dc0ff086a2703f5a03e4f82681c76c — Martin Angers 1 year, 10 months ago e2a8980
pkg/compiler: simple statements, more tests
42 files changed, 302 insertions(+), 7 deletions(-)

M pkg/compiler/ast.go
M pkg/compiler/parser.go
M pkg/compiler/parser_test.go
M pkg/compiler/printer.go
A pkg/compiler/testdata/parser/fn_assign.snow
A pkg/compiler/testdata/parser/fn_assign.snow.err
A pkg/compiler/testdata/parser/fn_assign.snow.want
A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow
A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.err
A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.want
A pkg/compiler/testdata/parser/fn_expr_stmt.snow
A pkg/compiler/testdata/parser/fn_expr_stmt.snow.err
A pkg/compiler/testdata/parser/fn_expr_stmt.snow.want
A pkg/compiler/testdata/parser/fn_mul.snow
A pkg/compiler/testdata/parser/fn_mul.snow.err
A pkg/compiler/testdata/parser/fn_mul.snow.want
A pkg/compiler/testdata/parser/fn_nested_blocks.snow
A pkg/compiler/testdata/parser/fn_nested_blocks.snow.err
A pkg/compiler/testdata/parser/fn_nested_blocks.snow.want
A pkg/compiler/testdata/parser/fn_nested_paren.snow
A pkg/compiler/testdata/parser/fn_nested_paren.snow.err
A pkg/compiler/testdata/parser/fn_nested_paren.snow.want
A pkg/compiler/testdata/parser/fn_nested_paren_newline.snow
A pkg/compiler/testdata/parser/fn_nested_unary.snow
A pkg/compiler/testdata/parser/fn_nested_unary.snow.err
A pkg/compiler/testdata/parser/fn_nested_unary.snow.want
A pkg/compiler/testdata/parser/fn_paren_expr.snow
A pkg/compiler/testdata/parser/fn_paren_expr.snow.err
A pkg/compiler/testdata/parser/fn_paren_expr.snow.want
A pkg/compiler/testdata/parser/fn_unary.snow
A pkg/compiler/testdata/parser/fn_unary.snow.err
A pkg/compiler/testdata/parser/fn_unary.snow.want
A pkg/compiler/testdata/parser/var_comma_for_semi.snow
A pkg/compiler/testdata/parser/var_comma_for_semi.snow.err
A pkg/compiler/testdata/parser/var_comma_for_semi.snow.want
A pkg/compiler/testdata/parser/var_invalid_type.snow
A pkg/compiler/testdata/parser/var_invalid_type.snow.err
A pkg/compiler/testdata/parser/var_invalid_type.snow.want
A pkg/compiler/testdata/parser/var_missing_type_init.snow.notyet
M pkg/compiler/visitor.go
M pkg/grammar/grammar.ebnf
A pkg/grammar/grammar.go
M pkg/compiler/ast.go => pkg/compiler/ast.go +19 -0
@@ 151,6 151,22 @@ type parenExpr struct {
func (p *parenExpr) Pos() gotoken.Pos { return p.lparen }
func (p *parenExpr) End() gotoken.Pos { return p.rparen + 1 }

type assignStmt struct {
	left   expr
	assign gotoken.Pos
	right  expr
}

func (a *assignStmt) Pos() gotoken.Pos { return a.left.Pos() }
func (a *assignStmt) End() gotoken.Pos { return a.right.End() }

type exprStmt struct {
	value expr
}

func (e *exprStmt) Pos() gotoken.Pos { return e.value.Pos() }
func (e *exprStmt) End() gotoken.Pos { return e.value.End() }

// a basic literal, e.g. 33 or "abc"
type basicLit struct {
	pos     gotoken.Pos


@@ 226,3 242,6 @@ func (f *fnDef) stmtNode()      {}
func (v *varDef) stmtNode()     {}
func (r *returnStmt) stmtNode() {}
func (b *badStmt) stmtNode()    {}
func (b *block) stmtNode()      {}
func (a *assignStmt) stmtNode() {}
func (e *exprStmt) stmtNode()   {}

M pkg/compiler/parser.go => pkg/compiler/parser.go +53 -2
@@ 64,6 64,16 @@ func (p *parser) errorExpected(pos gotoken.Pos, msg string) {
	p.error(pos, msg)
}

func (p *parser) safePos(pos gotoken.Pos) (res gotoken.Pos) {
	defer func() {
		if recover() != nil {
			res = gotoken.Pos(p.file.Base() + p.file.Size()) // EOF position
		}
	}()
	_ = p.file.Offset(pos) // trigger a panic if position is out-of-range
	return pos
}

func (p *parser) expect(tok token) gotoken.Pos {
	pos := p.pos
	if p.tok != tok {


@@ 92,6 102,7 @@ func (p *parser) expectSemi() {

// sets of tokens that can be sync'd to (in a call to p.advance)
var (
	// TopLevelStmt production in the grammar.
	topLevelStmtStart = map[token]bool{
		Var: true,
		Let: true,


@@ 132,6 143,14 @@ func (p *parser) parseStmt(topLevel bool) stmt {
		switch p.tok {
		case Return:
			return p.parseReturnStmt()
		case Lbrace:
			bk := p.parseBlock()
			p.expectSemi()
			return bk

			// tokens that may start a simple statement
		case Ident, Add, Sub, Lparen, Int, String:
			return p.parseSimpleStmt()
		}
	}



@@ 150,10 169,42 @@ func (p *parser) parseStmt(topLevel bool) stmt {
		}
		p.errorExpected(pos, lbl)
		p.advance(sync)
		return &badStmt{from: pos, to: p.pos}
		return &badStmt{from: pos, to: p.safePos(p.pos)}
	}
}

func (p *parser) parseSimpleStmt() stmt {
	pos := p.pos
	lhs := p.parseExpr()

	// check if this is an assignment statement
	if p.tok == Assign {
		opPos := p.expect(Assign)

		// for now, lhs must be an identifier, otherwise convert to a badExpr
		if _, ok := lhs.(*ident); !ok {
			p.errorExpected(pos, "identifier")
			lhs = &badExpr{
				from: lhs.Pos(),
				to:   lhs.End(),
			}
		}

		as := &assignStmt{
			left:   lhs,
			assign: opPos,
		}
		as.right = p.parseExpr()
		p.expectSemi()
		return as
	}

	// otherwise this is an expression statement
	es := &exprStmt{value: lhs}
	p.expectSemi()
	return es
}

func (p *parser) parseReturnStmt() *returnStmt {
	rs := &returnStmt{
		ret: p.expect(Return),


@@ 282,7 333,7 @@ func (p *parser) parseAtom() expr {
	default:
		pos := p.pos
		p.errorExpected(pos, "expression")
		return &badExpr{from: pos, to: p.pos}
		return &badExpr{from: pos, to: p.safePos(p.pos)}
	}
}


M pkg/compiler/parser_test.go => pkg/compiler/parser_test.go +2 -1
@@ 10,6 10,7 @@ import (
	"path/filepath"
	"testing"

	"github.com/kr/pretty"
	"github.com/kylelemons/godebug/diff"
)



@@ 44,7 45,7 @@ func TestParser(t *testing.T) {
			f := p.parseFile()
			p.errors.RemoveMultiples()
			parseErr := p.errors.Err()
			//pretty.Print(f)
			pretty.Print(f)

			var ebuf bytes.Buffer
			goscanner.PrintError(&ebuf, parseErr)

M pkg/compiler/printer.go => pkg/compiler/printer.go +4 -0
@@ 105,6 105,10 @@ func (p *printer) Visit(n node) visitor {
		p.printMsg(fmt.Sprintf("unary [%s]", n.op), true)
	case *parenExpr:
		p.printMsg("paren", true)
	case *assignStmt:
		p.printMsg("assign", true)
	case *exprStmt:
		p.printMsg("expr", true)
	case *basicLit:
		p.printMsg(fmt.Sprintf("%s [%s]", n.literal, n.value), true)
	case *ident:

A pkg/compiler/testdata/parser/fn_assign.snow => pkg/compiler/testdata/parser/fn_assign.snow +4 -0
@@ 0,0 1,4 @@
fn test(x: int) {
  var y: int
  y = x
}

A pkg/compiler/testdata/parser/fn_assign.snow.err => pkg/compiler/testdata/parser/fn_assign.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_assign.snow.want => pkg/compiler/testdata/parser/fn_assign.snow.want +14 -0
@@ 0,0 1,14 @@
file [0:40]
  fn [0:40]
    ident [test] [3:7]
    sig [7:15]
      param [8:14]
        ident [x] [8:9]
        ident [int] [11:14]
    block [16:40]
      var: [20:30]
        ident [y] [24:25]
        ident [int] [27:30]
      assign [33:38]
        ident [y] [33:34]
        ident [x] [37:38]

A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow => pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int, y: int) {
  "abc" = x + y
}

A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.err => pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.err +1 -0
@@ 0,0 1,1 @@
fn_assign_invalid_lhs.snow:2:3: expected identifier

A pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.want => pkg/compiler/testdata/parser/fn_assign_invalid_lhs.snow.want +16 -0
@@ 0,0 1,16 @@
file [0:43]
  fn [0:43]
    ident [test] [3:7]
    sig [7:23]
      param [8:15]
        ident [x] [8:9]
        ident [int] [11:14]
      param [16:22]
        ident [y] [16:17]
        ident [int] [19:22]
    block [24:43]
      assign [28:41]
        bad expr [28:33]
        binary [+] [36:41]
          ident [x] [36:37]
          ident [y] [40:41]

A pkg/compiler/testdata/parser/fn_expr_stmt.snow => pkg/compiler/testdata/parser/fn_expr_stmt.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int) {
  x + 3
}

A pkg/compiler/testdata/parser/fn_expr_stmt.snow.err => pkg/compiler/testdata/parser/fn_expr_stmt.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_expr_stmt.snow.want => pkg/compiler/testdata/parser/fn_expr_stmt.snow.want +12 -0
@@ 0,0 1,12 @@
file [0:27]
  fn [0:27]
    ident [test] [3:7]
    sig [7:15]
      param [8:14]
        ident [x] [8:9]
        ident [int] [11:14]
    block [16:27]
      expr [20:25]
        binary [+] [20:25]
          ident [x] [20:21]
          int [3] [24:25]

A pkg/compiler/testdata/parser/fn_mul.snow => pkg/compiler/testdata/parser/fn_mul.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int, y: int) -> int {
  return x * y
}

A pkg/compiler/testdata/parser/fn_mul.snow.err => pkg/compiler/testdata/parser/fn_mul.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_mul.snow.want => pkg/compiler/testdata/parser/fn_mul.snow.want +16 -0
@@ 0,0 1,16 @@
file [0:49]
  fn [0:49]
    ident [test] [3:7]
    sig [7:30]
      param [8:15]
        ident [x] [8:9]
        ident [int] [11:14]
      param [16:22]
        ident [y] [16:17]
        ident [int] [19:22]
      ident [int] [27:30]
    block [31:49]
      return [35:47]
        binary [*] [42:47]
          ident [x] [42:43]
          ident [y] [46:47]

A pkg/compiler/testdata/parser/fn_nested_blocks.snow => pkg/compiler/testdata/parser/fn_nested_blocks.snow +7 -0
@@ 0,0 1,7 @@
fn blocks() {
  {{
     {{
        {}
      }}
   }}
}

A pkg/compiler/testdata/parser/fn_nested_blocks.snow.err => pkg/compiler/testdata/parser/fn_nested_blocks.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_nested_blocks.snow.want => pkg/compiler/testdata/parser/fn_nested_blocks.snow.want +10 -0
@@ 0,0 1,10 @@
file [0:54]
  fn [0:54]
    ident [blocks] [3:9]
    sig [9:11]
    block [12:54]
      block [16:52]
        block [17:51]
          block [24:46]
            block [25:45]
              block [35:37]

A pkg/compiler/testdata/parser/fn_nested_paren.snow => pkg/compiler/testdata/parser/fn_nested_paren.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int) {
  (((x)))
}

A pkg/compiler/testdata/parser/fn_nested_paren.snow.err => pkg/compiler/testdata/parser/fn_nested_paren.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_nested_paren.snow.want => pkg/compiler/testdata/parser/fn_nested_paren.snow.want +13 -0
@@ 0,0 1,13 @@
file [0:29]
  fn [0:29]
    ident [test] [3:7]
    sig [7:15]
      param [8:14]
        ident [x] [8:9]
        ident [int] [11:14]
    block [16:29]
      expr [20:27]
        paren [20:27]
          paren [21:26]
            paren [22:25]
              ident [x] [23:24]

A pkg/compiler/testdata/parser/fn_nested_paren_newline.snow => pkg/compiler/testdata/parser/fn_nested_paren_newline.snow +6 -0
@@ 0,0 1,6 @@
# panics because of the semicolon insertion (after ident, then illegal ')', I believe?)
fn test(x: int) {
  (
   x
  )
}

A pkg/compiler/testdata/parser/fn_nested_unary.snow => pkg/compiler/testdata/parser/fn_nested_unary.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int) -> int {
  return -+-+--++-x
}

A pkg/compiler/testdata/parser/fn_nested_unary.snow.err => pkg/compiler/testdata/parser/fn_nested_unary.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_nested_unary.snow.want => pkg/compiler/testdata/parser/fn_nested_unary.snow.want +20 -0
@@ 0,0 1,20 @@
file [0:46]
  fn [0:46]
    ident [test] [3:7]
    sig [7:22]
      param [8:14]
        ident [x] [8:9]
        ident [int] [11:14]
      ident [int] [19:22]
    block [23:46]
      return [27:44]
        unary [-] [34:44]
          unary [+] [35:44]
            unary [-] [36:44]
              unary [+] [37:44]
                unary [-] [38:44]
                  unary [-] [39:44]
                    unary [+] [40:44]
                      unary [+] [41:44]
                        unary [-] [42:44]
                          ident [x] [43:44]

A pkg/compiler/testdata/parser/fn_paren_expr.snow => pkg/compiler/testdata/parser/fn_paren_expr.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int, y: int) -> int {
  return (x + -2) * (3 - (y/2))
}

A pkg/compiler/testdata/parser/fn_paren_expr.snow.err => pkg/compiler/testdata/parser/fn_paren_expr.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_paren_expr.snow.want => pkg/compiler/testdata/parser/fn_paren_expr.snow.want +26 -0
@@ 0,0 1,26 @@
file [0:66]
  fn [0:66]
    ident [test] [3:7]
    sig [7:30]
      param [8:15]
        ident [x] [8:9]
        ident [int] [11:14]
      param [16:22]
        ident [y] [16:17]
        ident [int] [19:22]
      ident [int] [27:30]
    block [31:66]
      return [35:64]
        binary [*] [42:64]
          paren [42:50]
            binary [+] [43:49]
              ident [x] [43:44]
              unary [-] [47:49]
                int [2] [48:49]
          paren [53:64]
            binary [-] [54:63]
              int [3] [54:55]
              paren [58:63]
                binary [/] [59:62]
                  ident [y] [59:60]
                  int [2] [61:62]

A pkg/compiler/testdata/parser/fn_unary.snow => pkg/compiler/testdata/parser/fn_unary.snow +3 -0
@@ 0,0 1,3 @@
fn test(x: int) {
  return -x
}

A pkg/compiler/testdata/parser/fn_unary.snow.err => pkg/compiler/testdata/parser/fn_unary.snow.err +0 -0
A pkg/compiler/testdata/parser/fn_unary.snow.want => pkg/compiler/testdata/parser/fn_unary.snow.want +11 -0
@@ 0,0 1,11 @@
file [0:31]
  fn [0:31]
    ident [test] [3:7]
    sig [7:15]
      param [8:14]
        ident [x] [8:9]
        ident [int] [11:14]
    block [16:31]
      return [20:29]
        unary [-] [27:29]
          ident [x] [28:29]

A pkg/compiler/testdata/parser/var_comma_for_semi.snow => pkg/compiler/testdata/parser/var_comma_for_semi.snow +1 -0
@@ 0,0 1,1 @@
var x: int,

A pkg/compiler/testdata/parser/var_comma_for_semi.snow.err => pkg/compiler/testdata/parser/var_comma_for_semi.snow.err +1 -0
@@ 0,0 1,1 @@
var_comma_for_semi.snow:1:11: expected ';', found ','

A pkg/compiler/testdata/parser/var_comma_for_semi.snow.want => pkg/compiler/testdata/parser/var_comma_for_semi.snow.want +4 -0
@@ 0,0 1,4 @@
file [0:10]
  var: [0:10]
    ident [x] [4:5]
    ident [int] [7:10]

A pkg/compiler/testdata/parser/var_invalid_type.snow => pkg/compiler/testdata/parser/var_invalid_type.snow +1 -0
@@ 0,0 1,1 @@
var x: 123

A pkg/compiler/testdata/parser/var_invalid_type.snow.err => pkg/compiler/testdata/parser/var_invalid_type.snow.err +1 -0
@@ 0,0 1,1 @@
var_invalid_type.snow:1:8: expected 'ident', found 123

A pkg/compiler/testdata/parser/var_invalid_type.snow.want => pkg/compiler/testdata/parser/var_invalid_type.snow.want +4 -0
@@ 0,0 1,4 @@
file [0:7]
  var: [0:7]
    ident [x] [4:5]
    ident [] [7:7]

A pkg/compiler/testdata/parser/var_missing_type_init.snow.notyet => pkg/compiler/testdata/parser/var_missing_type_init.snow.notyet +1 -0
@@ 0,0 1,1 @@
var x

M pkg/compiler/visitor.go => pkg/compiler/visitor.go +13 -0
@@ 86,6 86,19 @@ func walk(v visitor, node node) {
			walk(v, n.value)
		}

	case *assignStmt:
		if n.left != nil {
			walk(v, n.left)
		}
		if n.right != nil {
			walk(v, n.right)
		}

	case *exprStmt:
		if n.value != nil {
			walk(v, n.value)
		}

	case *commentGroup:
		for _, comment := range n.list {
			walk(v, comment)

M pkg/grammar/grammar.ebnf => pkg/grammar/grammar.ebnf +15 -4
@@ 1,31 1,42 @@
// NOTE: comment production omitted for brevity.

SourceFile = { TopLevelStmt } .
TopLevelStmt = VarDef | FuncDef .

// TODO(VarDef): eventually support groups of var/let definitions.
// TODO(ArgsList): eventually support grouping multiple argument names of same type
// TODO: type is not just an identifier, more complex (e.g. func type, tuple type, generics, etc.)
// TODO(Stmt): for now, limit to variable definitions and return expressions.
// TODO(string): support a raw string mode, escapes like Go? or an extendable prefix-based literal? (e.g. r"a\b\c")
// TODO: char/rune literal type?

TopLevelStmt = VarDef | FuncDef .
Stmt = TopLevelStmt | SimpleStmt | Block | ReturnStmt .
SimpleStmt = ExprStmt | AssignStmt .

// => Variable definition statement
VarDef = LetVarDef | VarVarDef .
LetVarDef = "let" ident ( TypedVarDef | InferredVarDef ) .
VarVarDef = "var" ident ( TypedVarDef | InferredVarDef ) .
TypedVarDef = ":" ident [ InferredVarDef ] .
InferredVarDef = "=" Expr .

// => Function definition statement
FuncDef = "fn" ident ParamsList [ FuncTrailer ] Block .
ParamsList = "(" [ ParamDef { "," ParamDef } [ "," ] ] ")" .
ParamDef = ident ":" ident .
FuncTrailer = "->" ident .

// => Assignment statement 
AssignStmt = ident "=" Expr .

// => Block statement
Block = "{" { Stmt } "}" .

// TODO: expression statement
Stmt = VarDef | ReturnStmt .
// => Return statement
ReturnStmt = "return" [ Expr ] .

// => Expression statement 
ExprStmt = Expr .

Expr = Term { ( "+" | "-" ) Term } .
Term = Factor { ( "*" | "/" | "%" ) Factor } .
Factor = ( "+" | "-" ) Factor | Atom .

A pkg/grammar/grammar.go => pkg/grammar/grammar.go +6 -0
@@ 0,0 1,6 @@
package grammar

// just so that test coverage for this package returns 100% and not [no statements]
var _ = func() int {
	return 1
}()