~thon/thon

ee41ae62385bc1d0a93f81931749300571b02baa — Evan Bergeron 3 months ago 3fb8a37
First lexing regression test and eatKeywordOrName
3 files changed, 27 insertions(+), 24 deletions(-)

A examples/lex00.thon
M lex.sml
M thon.sml
A examples/lex00.thon => examples/lex00.thon +4 -0
@@ 0,0 1,4 @@
fun foo(a nat):
    fun bar(n nat):
        return n
    return a

M lex.sml => lex.sml +18 -24
@@ 1,10 1,10 @@
structure Lex : sig
datatype Token = FUN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT
datatype Token = FUN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN
val lexFile : string -> Token list
end  =
struct

datatype Token = FUN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT
datatype Token = FUN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN


exception UnexpectedIndentLevel


@@ 53,7 53,18 @@ fun eatWord w s = (
    eatWhitespace s
)

fun lexLines' s out indentLevel =
fun eatKeywordOrName (w, tok) s indentLevel out =
    if onKeyword w s then (
        eatWord w s;
        lexLines' s (tok::out) indentLevel
    ) else (
        let val name = getName s in
            eatWord name s;
            lexLines' s ((NAME name)::out) indentLevel
        end
    )

and lexLines' s out indentLevel =
    case lookaheadN s 1 of
        "" => out
      | " " =>


@@ 66,31 77,14 @@ fun lexLines' s out indentLevel =
                (eatWord spaces s;
                 lexLines' s out indentLevel)
            else if lookaheadN s (4 * ((!indentLevel)-1)) = dedentSpaces then
                (eatWord spaces s;
                (eatWord dedentSpaces s;
                 lexLines' s (DEDENT::out) indentLevel)
            else
                raise UnexpectedIndentLevel
        end
      | "f" =>
        if onKeyword "fun" s then (
            eatWord "fun" s;
            lexLines' s (FUN::out) indentLevel
        ) else (
            let val name = getName s in
                eatWord name s;
                lexLines' s ((NAME name)::out) indentLevel
            end
        )
      | "n" =>
        if onKeyword "nat" s then (
            eatWord "nat" s;
            lexLines' s (NAT::out) indentLevel
        ) else (
            let val name = getName s in
                eatWord name s;
                lexLines' s ((NAME name)::out) indentLevel
            end
        )
      | "f" => eatKeywordOrName ("fun", FUN) s indentLevel out
      | "n" => eatKeywordOrName ("nat", NAT) s indentLevel out
      | "r" => eatKeywordOrName ("return", RETURN) s indentLevel out
      | "(" => (
          eatWord "(" s;
          lexLines' s (LPAREN::out) indentLevel

M thon.sml => thon.sml +5 -0
@@ 1235,6 1235,11 @@ val autoDatatype = elaborateDatatypes (parse "data List = Nil unit | Cons nat * 
val Zero = runFile "/home/evan/thon/examples/auto-natlist.thon";
val Succ (Succ Zero) = runFile "/home/evan/thon/examples/bst-depth.thon";

(* Handwritten lexer tests *)
open Lex;
val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,INDENT,FUN,NAME "bar",LPAREN,
     NAME "n",NAT,RPAREN,INDENT,RETURN,NAME "n",DEDENT,RETURN,NAME "a"] = Lex.lexFile "/home/evan/thon/examples/lex00.thon";

in
()
end