~thon/thon

55b407fb32fc7e6fcd49d2e52432d3fbbd244bdd — Evan Bergeron 3 months ago 20479f0
Lex explicit newline tokens
2 files changed, 33 insertions(+), 24 deletions(-)

M lex.sml
M thon.sml
M lex.sml => lex.sml +10 -4
@@ 1,10 1,10 @@
structure Lex : sig
datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA
datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA | NEWLINE
val lexFile : string -> Token list
end  =
struct

datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA
datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA | NEWLINE

exception UnexpectedIndentLevel
exception UnexpectedToken


@@ 44,7 44,7 @@ fun eatWhitespace stream =
    case TextIO.lookahead stream of
        NONE => ()
      (* If I need to spit out NEWLINE tokens, feed `out` down here *)
      | SOME #"\n" => (TextIO.input1 stream; ())
      | SOME #"\n" => ((*TextIO.input1 stream*)(); ())
      | SOME c => if (Char.isSpace c)then
                  (TextIO.input1 stream; eatWhitespace stream)
                  else ()


@@ 75,6 75,7 @@ fun eatKeywordOrName (w, tok) s indentLevel out =
    )

and lexLines' s out indentLevel =
    (();
    case lookaheadN s 1 of
        "" => out
      | " " =>


@@ 94,6 95,10 @@ and lexLines' s out indentLevel =
            else
                raise UnexpectedIndentLevel
        end
      | "\n" => (
          TextIO.input1 s; (* can't eatWord here - keep leading spaces *)
          lexLines' s (NEWLINE::out) indentLevel
        )
      | "=" =>
        if onKeyword "=>" s then (
            eatWord "=>" s;


@@ 144,7 149,7 @@ and lexLines' s out indentLevel =
              (eatWord ":\n" s;
               (* could also incr after checking next line *)
               indentLevel := !indentLevel + 1;
               lexLines' s (INDENT::out) indentLevel)
               lexLines' s (INDENT::NEWLINE::out) indentLevel)
          else
              (eatWord ":" s;
               lexLines' s (COLON::out) indentLevel)


@@ 153,6 158,7 @@ and lexLines' s out indentLevel =
                     eatWord name s;
                     lexLines' s ((NAME name)::out) indentLevel
                 end
                     )

fun lexLines s indentLevel =
    let val backwards = lexLines' s [] indentLevel in List.rev backwards end

M thon.sml => thon.sml +23 -20
@@ 1237,29 1237,32 @@ val Succ (Succ Zero) = runFile "/home/evan/thon/examples/bst-depth.thon";

(* Handwritten lexer tests *)
open Lex;
val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,INDENT,FUN,NAME "bar",LPAREN,
     NAME "n",NAT,RPAREN,INDENT,RETURN,NAME "n",DEDENT,RETURN,NAME "a"] = Lex.lexFile "/home/evan/thon/examples/lex00.thon";

val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,INDENT,LET,NAME "x",NAT,
   SARROW,NAT,EQUAL,FN,LPAREN,NAME "x",NAT,RPAREN,NAT,DARROW,SUCC,NAME "x",
   FUN,NAME "bar",LPAREN,NAME "n",NAT,RPAREN,NAT,INDENT,RETURN,NAME "n",
   DEDENT,RETURN,NAME "a"] : Lex.Token list =
val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NEWLINE,INDENT,FUN,NAME "bar",LPAREN,
     NAME "n",NAT,RPAREN,NEWLINE,INDENT,RETURN,NAME "n",NEWLINE,DEDENT,RETURN,NAME "a",
     NEWLINE]
    = Lex.lexFile "/home/evan/thon/examples/lex00.thon";

val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,NEWLINE,INDENT,LET,NAME "x",
   NAT,SARROW,NAT,EQUAL,FN,LPAREN,NAME "x",NAT,RPAREN,NAT,DARROW,SUCC,
   NAME "x",NEWLINE,FUN,NAME "bar",LPAREN,NAME "n",NAT,RPAREN,NAT,NEWLINE,
   INDENT,RETURN,NAME "n",NEWLINE,DEDENT,RETURN,NAME "a",NEWLINE]
  : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex01.thon";

val
  [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,INDENT,LET,NAME "b",NAT,
   EQUAL,ZERO,LET,NAME "f",NAT,SARROW,NAT,EQUAL,FN,LPAREN,NAME "x",NAT,RPAREN,
   NAT,DARROW,SUCC,NAME "x",IF,NAME "t",INDENT,LET,NAME "c",NAT,EQUAL,
   NAME "f",LPAREN,NAME "b",RPAREN,DEDENT,ELSE,INDENT,LET,NAME "c",NAT,EQUAL,
   NAME "f",LPAREN,NAME "a",RPAREN,DEDENT,LET,NAME "p",LPAREN,NAT,COMMA,NAT,
   RPAREN,EQUAL,LPAREN,SUCC,ZERO,COMMA,ZERO,RPAREN,DATA,NAME "tree",EQUAL,
   NAME "nil",BAR,NAME "node",NAT,NAME "tree",NAME "tree",LET,NAME "n",
   NAME "tree",EQUAL,NAME "nil",LET,NAME "n2",NAME "tree",EQUAL,NAME "node",
   LPAREN,ZERO,COMMA,NAME "nil",COMMA,NAME "nil",RPAREN,CASE,NAME "n2",INDENT,
   NAME "nil",INDENT,RETURN,NAME "b",DEDENT,NAME "node",LPAREN,NAME "val",
   COMMA,NAME "l",COMMA,NAME "r",RPAREN,INDENT,RETURN,NAME "f",LPAREN,
   NAME "n",RPAREN,DEDENT,RETURN,NAME "TODOstillNeedToHandleMultiDedent",
   DEDENT,RETURN,NAME "b"] : Lex.Token list =
  [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,NEWLINE,INDENT,LET,NAME "b",NAT,
   EQUAL,ZERO,NEWLINE,LET,NAME "f",NAT,SARROW,NAT,EQUAL,FN,LPAREN,NAME "x",NAT,RPAREN,
   NAT,DARROW,SUCC,NAME "x",NEWLINE,IF,NAME "t",NEWLINE,INDENT,LET,NAME "c",NAT,EQUAL,
   NAME "f",LPAREN,NAME "b",RPAREN,NEWLINE,DEDENT,ELSE,NEWLINE,INDENT,LET,NAME "c",NAT,EQUAL,
   NAME "f",LPAREN,NAME "a",RPAREN,NEWLINE,DEDENT,LET,NAME "p",LPAREN,NAT,COMMA,NAT,
   RPAREN,EQUAL,LPAREN,SUCC,ZERO,COMMA,ZERO,RPAREN,NEWLINE,DATA,NAME "tree",EQUAL,
   NAME "nil",BAR,NAME "node",NAT,NAME "tree",NAME "tree",NEWLINE,LET,NAME "n",
   NAME "tree",EQUAL,NAME "nil",NEWLINE,LET,NAME "n2",NAME "tree",EQUAL,NAME "node",
   LPAREN,ZERO,COMMA,NAME "nil",COMMA,NAME "nil",RPAREN,NEWLINE,CASE,NAME "n2",NEWLINE,INDENT,
   NAME "nil",NEWLINE,INDENT,RETURN,NAME "b",NEWLINE,DEDENT,NAME "node",LPAREN,NAME "val",
   COMMA,NAME "l",COMMA,NAME "r",RPAREN,NEWLINE,INDENT,RETURN,NAME "f",LPAREN,
   NAME "n",RPAREN,NEWLINE,DEDENT,RETURN,NAME "TODOstillNeedToHandleMultiDedent",
   NEWLINE,DEDENT,RETURN,NAME "b",NEWLINE] : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex02.thon";