~thon/thon

8ccc530dbf10c37d4376fce4b1dac36119c053dd — Evan Bergeron 3 months ago e39f9a8
Refactor lexing dedent/indent still doesn't handle empty lines correctly
4 files changed, 65 insertions(+), 31 deletions(-)

M examples/lex03.thon
M lex.sml
M parse.sml
M thon.sml
M examples/lex03.thon => examples/lex03.thon +1 -1
@@ 1,2 1,2 @@
fun foo(420 nat):
    return 420
    420

M lex.sml => lex.sml +41 -25
@@ 12,6 12,12 @@ exception UnexpectedIndentLevel
exception UnexpectedToken of string
exception UnimplementTokenToString

fun println s = print (s  ^ "\n")

fun debugPrint s =
    if false then println s
    else ()

fun tokenToString FUN = "FUN"
  | tokenToString FN = "FN"
  | tokenToString NAT = "NAT"


@@ 53,20 59,25 @@ fun lookaheadOnlyN s n =
    in (List.nth (chars, (List.length chars) - 1)) end

fun getName' s n =
    (debugPrint ("getName " ^ (Int.toString n));
    if not (Char.isAlphaNum (lookaheadOnlyN s n)) then
        lookaheadN s (n-1)
    else
        getName' s (n+1)
        getName' s (n+1))

fun getName s = getName' s 1

fun getNumSpaces' s n =
    if not (#" " = (lookaheadOnlyN s n)) then
        n
    else
        getNumSpaces' s (n+1)
fun eatAndGetNumSpaces' s n =
    (debugPrint ("eatAndGetNumSpaces' " ^ (Int.toString n));
     case TextIO.lookahead s of
         SOME #" " => (
          TextIO.input1 s;
          eatAndGetNumSpaces' s (n+1)
         )
       | _ => n
    )

fun getNumSpaces s = getNumSpaces' s 0
fun eatAndGetNumSpaces s = eatAndGetNumSpaces' s 0

fun eatWhitespace stream =
    case TextIO.lookahead stream of


@@ 78,12 89,13 @@ fun eatWhitespace stream =
                  else ()

fun onKeyword kw s =
    (debugPrint ("onKeyword" ^ kw);
    let val prefixOk = kw = (lookaheadN s (String.size kw))
        val afterChar = lookaheadOnlyN s ((String.size kw)+1)
        val suffixOk = not (Char.isAlphaNum afterChar)
    in
        prefixOk andalso suffixOk
    end
    end)

fun eatWord w s = (
    TextIO.inputN (s, (String.size w));


@@ 103,25 115,30 @@ fun eatKeywordOrName (w, tok) s indentLevel out =
    )

and lexLines' s out indentLevel =
    (();
    (debugPrint "=======================";
     List.map (fn tok => debugPrint (tokenToString tok)) out;
    case lookaheadN s 1 of
        "" => out
      | " " =>
        let val spaces =
                String.concat (List.tabulate (4*(!indentLevel), fn _ => " "))
            val dedentSpaces =
                String.concat (List.tabulate (4*((!indentLevel)-1), fn _ => " "))
        let
            (* TODO assert last elt of out is NEWLINE here *)
            val () = debugPrint ("Indent level: " ^ (Int.toString (!indentLevel)))
            val numSpaces = eatAndGetNumSpaces s
            (* UNDONE 2 space indent *)
            val thisLineIndentLevel = numSpaces div 4;
            val () = debugPrint ("thisLineIndentLevel " ^ (Int.toString (thisLineIndentLevel)))
            val tok = if thisLineIndentLevel > (!indentLevel) then INDENT else DEDENT
            val numToks = abs (thisLineIndentLevel - (!indentLevel))
            val toks = List.tabulate (numToks, fn _ => tok);
        in
            (* TODO can dedent unboundedly - use getNumSpaces *)
            if lookaheadN s (4 * (!indentLevel)) = spaces then
                (eatWord spaces s;
                 lexLines' s out indentLevel)
            else if lookaheadN s (4 * ((!indentLevel)-1)) = dedentSpaces then
                (eatWord dedentSpaces s;
                 indentLevel := !indentLevel - 1;
                 lexLines' s (DEDENT::out) indentLevel)
            (
            if thisLineIndentLevel = (!indentLevel) then
                (* No indent or dedent here *)
                lexLines' s out indentLevel
            else
                raise UnexpectedIndentLevel
                 (indentLevel := thisLineIndentLevel;
                 lexLines' s (toks @ out) indentLevel)
            )
        end
      | "\n" => (
          TextIO.input1 s; (* can't eatWord here - keep leading spaces *)


@@ 174,10 191,9 @@ and lexLines' s out indentLevel =
      )
      | ":" => (
          if lookaheadN s 2 = ":\n" then
              (eatWord ":\n" s;
              (eatWord ":" s;
               (* could also incr after checking next line *)
               indentLevel := !indentLevel + 1;
               lexLines' s (INDENT::NEWLINE::out) indentLevel)
               lexLines' s out indentLevel)
          else
              (eatWord ":" s;
               lexLines' s (COLON::out) indentLevel)

M parse.sml => parse.sml +10 -5
@@ 11,7 11,7 @@ fun incr i = (i := !i + 1)
fun println s = print (s  ^ "\n")

fun debugPrint s =
    if true then println s
    if false then println s
    else ()

fun errMsg (expectedToken, actualToken) =


@@ 47,9 47,14 @@ fun consumeNewlines tokens i =
      | _ => ()

fun parseType tokens i =
    case List.nth (tokens, !i) of
        Lex.NAT => (i := (!i) + 1; A.Nat)
      | _ => raise Unimplemented
    let val this =
            (case List.nth (tokens, !i) of
                 Lex.NAT => (i := (!i) + 1; A.Nat)
               | _ => raise Unimplemented)
    in
        (case List.nth (tokens, !i) of
             Lex.SARROW => (incr(i); A.Arr(this, (parseType tokens i))) | _ => this)
    end

fun parseExpr tokens i =
    (if (!i) >= (List.length tokens) then A.TmUnit else


@@ 59,7 64,7 @@ fun parseExpr tokens i =
              val () = expect tokens Lex.FUN i
              val funcName = consumeName tokens i
              val () = expect tokens Lex.LPAREN i
              (* TODO multiple params *)
              (* TODO multiple params - should implement n-nary products first *)
              val argName = consumeName tokens i
              val argType = parseType tokens i
              val () = expect tokens Lex.RPAREN i

M thon.sml => thon.sml +13 -0
@@ 1291,6 1291,19 @@ val Let
     App (Var ("ident",0),Zero)) : Ast.exp =
    newParseFile "/home/evan/thon/examples/parse01.thon";


val Let
    ("foo",Arr (Nat,Arr (Nat,Nat)),
     Fix
       ("foo",Arr (Nat,Arr (Nat,Nat)),
        Fn
          ("a",Nat,
           Let
             ("bar",Arr (Nat,Nat),
              Fix ("bar",Arr (Nat,Nat),Fn ("b",Nat,Var ("a",2))),TmUnit))),
     TmUnit) : Ast.exp =
    newParseFile "/home/evan/thon/examples/parse02.thon";

in
()
end