~thon/thon

f20d5ac2b2d2af3dac4ac1a4c3ef22f37f2d3b39 — Evan Bergeron 3 months ago 55b407f
Print lexing error msg, don't allow indentifiers starting with numberic character
4 files changed, 26 insertions(+), 9 deletions(-)

A examples/lex01.thon
A examples/lex03.thon
M lex.sml
M thon.sml
A examples/lex01.thon => examples/lex01.thon +5 -0
@@ 0,0 1,5 @@
fun foo(a nat) nat:
    let x nat -> nat = fn(x nat) nat => s x
    fun bar(n nat) nat:
        return n
    return a

A examples/lex03.thon => examples/lex03.thon +2 -0
@@ 0,0 1,2 @@
fun foo(420 nat):
    return 420

M lex.sml => lex.sml +18 -9
@@ 1,13 1,14 @@
structure Lex : sig
datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA | NEWLINE
val lexFile : string -> Token list
val lexFileNoPrintErrMsg : string -> Token list
end  =
struct

datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA | NEWLINE

exception UnexpectedIndentLevel
exception UnexpectedToken
exception UnexpectedToken of string

fun lookaheadN s n =
    (* Can raise Size *)


@@ 107,7 108,7 @@ and lexLines' s out indentLevel =
            eatWord "=" s;
            lexLines' s (EQUAL::out) indentLevel
        ) else (
            raise UnexpectedToken
            raise UnexpectedToken("saw `=`, expected `=>` or `=`")
        )
      | "i" => eatKeywordOrName ("if", IF) s indentLevel out
      | "t" => eatKeywordOrName ("then", THEN) s indentLevel out


@@ 154,24 155,32 @@ and lexLines' s out indentLevel =
              (eatWord ":" s;
               lexLines' s (COLON::out) indentLevel)
      )
      | other => let val name = getName s in
                     eatWord name s;
                     lexLines' s ((NAME name)::out) indentLevel
                 end
                     )
      | other =>
        if not (Char.isAlpha (String.sub (other, 0))) then
            raise UnexpectedToken("indentifiers must start with alphabetic characters")
        else
            let val name = getName s in
            eatWord name s;
            lexLines' s ((NAME name)::out) indentLevel
            end
      )

fun lexLines s indentLevel =
    let val backwards = lexLines' s [] indentLevel in List.rev backwards end

fun lex s =
fun lex s printErrMsg =
    let
        val indentLevel = ref 0;
        val forewards = lexLines s indentLevel
    in
        forewards
    end
    handle UnexpectedToken msg => (if printErrMsg then print ("Syntax error: " ^ msg ^ "\n") else ();
                                   raise (UnexpectedToken msg) )

fun lexFile filename = lex (TextIO.openIn filename) true

fun lexFile filename = lex (TextIO.openIn filename)
fun lexFileNoPrintErrMsg filename = lex (TextIO.openIn filename) false


end

M thon.sml => thon.sml +1 -0
@@ 1265,6 1265,7 @@ val
   NEWLINE,DEDENT,RETURN,NAME "b",NEWLINE] : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex02.thon";

val true = (Lex.lexFileNoPrintErrMsg "/home/evan/thon/examples/lex03.thon"; false) handle UnexpectedToken => true;

in
()