~thon/thon

b49f512482ff7627b984d9d6ec7b63ede3839b69 — Evan Bergeron 2 months ago 4dada47
Handle consecutive dedents
4 files changed, 56 insertions(+), 19 deletions(-)

A examples/parse02.thon
M lex.sml
M parse.sml
M thon.sml
A examples/parse02.thon => examples/parse02.thon +3 -0
@@ 0,0 1,3 @@
fun foo(a nat) nat -> nat:
    fun bar(b nat) nat:
        a

M lex.sml => lex.sml +12 -9
@@ 8,6 8,7 @@ struct

datatype Token = FUN | FN | NAT | COLON | LPAREN | RPAREN | NAME of string | INDENT | DEDENT | RETURN | ZERO | SUCC | LET | SARROW | EQUAL | DARROW | IF | THEN | ELSE | DATA | BAR | CASE | COMMA | NEWLINE

exception No
exception UnexpectedIndentLevel
exception UnexpectedToken of string
exception UnimplementTokenToString


@@ 140,22 141,25 @@ and lexLines' s out indentLevel =
     List.map (fn tok => debugPrint (tokenToString tok)) out;
    case lookaheadN s 1 of
        "" => out
      | " " => 
      | " " => raise No
      | "\n" => (
     (case lookaheadOnlyN s 2 of
         SOME #"\n" => (TextIO.input1 s;
                        lexLines' s (NEWLINE::out) indentLevel)
        | _ =>
          (TextIO.input1 s; (* can't eatWord here - keep leading spaces *)
        let val toks = getIndentDedentTokens s out indentLevel
        in
            if List.length toks = 0 then
                (* No indent or dedent here *)
                lexLines' s out indentLevel
                lexLines' s (NEWLINE::out) indentLevel
            else
                (indentLevel := !indentLevel +
                                ((List.length toks) *
                                 (if DEDENT = List.nth (toks, 0) then ~1 else 1))
                ; lexLines' s (toks @ out) indentLevel)
        end
      | "\n" => (
          TextIO.input1 s; (* can't eatWord here - keep leading spaces *)
          lexLines' s (NEWLINE::out) indentLevel
        )
                ; lexLines' s (toks @ (NEWLINE::out)) indentLevel)
        end)
        ))
      | "=" =>
        if onKeyword "=>" s then (
            eatWord "=>" s;


@@ 204,7 208,6 @@ and lexLines' s out indentLevel =
      | ":" => (
          if lookaheadN s 2 = ":\n" then
              (eatWord ":" s;
               (* could also incr after checking next line *)
               lexLines' s out indentLevel)
          else
              (eatWord ":" s;

M parse.sml => parse.sml +29 -5
@@ 63,6 63,7 @@ fun parseExpr tokens i =
          let
              val () = expect tokens Lex.FUN i
              val funcName = consumeName tokens i
              val () = debugPrint (funcName ^ " begin")
              val () = expect tokens Lex.LPAREN i
              (* TODO multiple params - should implement n-nary products first *)
              val argName = consumeName tokens i


@@ 72,13 73,35 @@ fun parseExpr tokens i =
              val funcType = A.Arr(argType, retType)
              val () = consumeNewlines tokens i
              val () = expect tokens Lex.INDENT i
                                       val () = debugPrint "func ident"
              val () = debugPrint (funcName ^ " indent")
              val body = parseExpr tokens i
              val () = debugPrint (funcName ^ " end of body")
              val () = consumeNewlines tokens i
              val rest = parseExpr tokens i
              val () = expect tokens Lex.DEDENT i
              val () = debugPrint (funcName ^ " dedent")
              val () = consumeNewlines tokens i
              val () = debugPrint (funcName ^ " afterwards")
          in
              A.Let(funcName, funcType,
                    A.Fix(funcName, funcType, A.Fn(argName, argType, body)), rest)
              if (!i) < (List.length tokens) andalso
                 List.nth (tokens, (!i))  = Lex.DEDENT then
                  (debugPrint (funcName ^ "see dedent next");
                   (* TODO double check these semantics. If there's a
                      dedent after this funciton definition, then this is
                      the last chunk of the parent block and so the value
                      of the parent block should be this function? If so,
                      will need to replicate this logic across every
                      other construct. *)
                   A.Let(funcName, funcType,
                         A.Fix(funcName, funcType,
                               A.Fn(argName, argType, body)), A.Var(funcName, ~1)))
              else
                  let
                      val rest = parseExpr tokens i
                  in
                      A.Let(funcName, funcType,
                            A.Fix(funcName, funcType,
                                  A.Fn(argName, argType, body)), rest)
                  end
          end
        | Lex.ZERO => (incr(i); A.Zero)
        | Lex.NAME name =>


@@ 100,7 123,8 @@ fun parseExpr tokens i =
                    end
          )

        | tok => (println (Lex.tokenToString tok); raise Unimplemented))
        | tok => (println ("Got unexpected " ^
                           (Lex.tokenToString tok)); raise Unimplemented))
    )

fun parseFile filename =

M thon.sml => thon.sml +12 -5
@@ 1253,13 1253,13 @@ val Succ (Succ Zero) = runFile "/home/evan/thon/examples/bst-depth.thon";
open Lex;
val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NEWLINE,INDENT,FUN,NAME "bar",LPAREN,
     NAME "n",NAT,RPAREN,NEWLINE,INDENT,RETURN,NAME "n",NEWLINE,DEDENT,RETURN,NAME "a",
     NEWLINE]
     NEWLINE,DEDENT] : Lex.Token list
    = Lex.lexFile "/home/evan/thon/examples/lex00.thon";

val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,NEWLINE,INDENT,LET,NAME "x",
   NAT,SARROW,NAT,EQUAL,FN,LPAREN,NAME "x",NAT,RPAREN,NAT,DARROW,SUCC,
   NAME "x",NEWLINE,FUN,NAME "bar",LPAREN,NAME "n",NAT,RPAREN,NAT,NEWLINE,
   INDENT,RETURN,NAME "n",NEWLINE,DEDENT,RETURN,NAME "a",NEWLINE]
   INDENT,RETURN,NAME "n",NEWLINE,DEDENT,RETURN,NAME "a",NEWLINE,DEDENT]
  : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex01.thon";



@@ 1276,7 1276,7 @@ val
   NAME "nil",NEWLINE,INDENT,RETURN,NAME "b",NEWLINE,DEDENT,NAME "node",LPAREN,NAME "val",
   COMMA,NAME "l",COMMA,NAME "r",RPAREN,NEWLINE,INDENT,RETURN,NAME "f",LPAREN,
   NAME "n",RPAREN,NEWLINE,DEDENT,RETURN,NAME "TODOstillNeedToHandleMultiDedent",
   NEWLINE,DEDENT,RETURN,NAME "b",NEWLINE] : Lex.Token list =
   NEWLINE,DEDENT,RETURN,NAME "b",NEWLINE,DEDENT] : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex02.thon";

val true = (Lex.lexFileNoPrintErrMsg "/home/evan/thon/examples/lex03.thon"; false) handle UnexpectedToken => true;


@@ 1292,6 1292,12 @@ val Let
    newParseFile "/home/evan/thon/examples/parse01.thon";



val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,SARROW,NAT,NEWLINE,INDENT,
   FUN,NAME "bar",LPAREN,NAME "b",NAT,RPAREN,NAT,NEWLINE,INDENT,NAME "a",
   NEWLINE,DEDENT,DEDENT] : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/parse02.thon";

val Let
    ("foo",Arr (Nat,Arr (Nat,Nat)),
     Fix


@@ 1300,14 1306,15 @@ val Let
          ("a",Nat,
           Let
             ("bar",Arr (Nat,Nat),
              Fix ("bar",Arr (Nat,Nat),Fn ("b",Nat,Var ("a",2))),TmUnit))),
              Fix ("bar",Arr (Nat,Nat),Fn ("b",Nat,Var ("a",2))),
              Var ("bar",0)))),
     TmUnit) : Ast.exp =
    newParseFile "/home/evan/thon/examples/parse02.thon";


val [FUN,NAME "foo",LPAREN,NAME "a",NAT,RPAREN,NAT,SARROW,NAT,NEWLINE,INDENT,
   FUN,NAME "bar",LPAREN,NAME "b",NAT,RPAREN,NAT,NEWLINE,INDENT,NAME "a"]
  : Lex.Token list = 
  : Lex.Token list =
    Lex.lexFile "/home/evan/thon/examples/lex04.thon";

in