~tim/scheme-vm

223dd4a8c9cd2e0c4994af71a6edbc99b466bd44 — Tim Morgan 4 years ago ed6d292
Hook Rust parser into Ruby

I have a bunch of failing tests, so I've focused the parser test for now.
8 files changed, 53 insertions(+), 193 deletions(-)

M Cargo.toml
M parser.rb
M spec/parser_spec.rb
M src/lib.rs
M src/lisp.rustpeg
A src/quotes.rs
D src/tests.rs
D test.rb
M Cargo.toml => Cargo.toml +1 -0
@@ 10,6 10,7 @@ crate-type = ["dylib"]
[dependencies]
libc = "0.2.x"
ruby-sys = "0.2.20"
lazy_static = "0.2.1"

[build-dependencies]
peg = { git = "https://github.com/kevinmehall/rust-peg", branch = "master" }

M parser.rb => parser.rb +14 -111
@@ 1,126 1,29 @@
require 'fiddle'
require_relative 'vm/atom'
require 'parslet'

module LISP
  class Parser < Parslet::Parser
    rule(:whitespace) do
      match('[ \t\n]').repeat
    end

    rule(:escape) do
      str('\\') >> any
    end

    rule(:string) do
      (str('"') >> (match('[^"]') | escape).repeat >> str('"')).as(:string)
    end

    rule(:atom) do
      quote.maybe >> (
        str('|') >> match('[^|]').repeat(1).as(:atom) >> str('|') |
        match('[^\(\) \t\n\[\]\{\}]').repeat(1).as(:atom)
      )
    end

    rule(:sexp) do
      quoted_sexp | simple_sexp
    end

    rule(:quote) do
      (str("'") | str(',@') | str(',') | str('`')).as(:quote)
    end

    rule(:quoted_sexp) do
      quote >> simple_sexp
    end

    rule(:simple_sexp) do
      (str('(') >> expression.repeat >> str(')')).as(:sexp)
    end

    rule(:comment) do
      block_comment | line_comment | datum_comment.as(:comment)
    end

    rule(:block_comment) do
      str('#|') >> (str('|#').absent? >> any).repeat >> str('|#')
    end

    rule(:line_comment) do
      str(';') >> match('[^\n]').repeat
    end

    rule(:datum_comment) do
      str('#;') >> str(' ').maybe >> (atom | sexp)
    end

    rule(:expression) do
      (string | comment | atom | sexp) >> whitespace
    end

    rule(:program) do
      expression.repeat(1)
    end

    root(:program)
  end

  class Transform < Parslet::Transform
    QUOTE_METHOD = {
      "'"  => 'quote',
      ',@' => 'unquote-splicing',
      ','  => 'unquote',
      '`'  => 'quasiquote'
    }.freeze

    rule(comment: subtree(:comment))

    rule(quote: simple(:quote), atom: simple(:atom)) do
      method = QUOTE_METHOD[quote.to_s]
      (line, column) = atom.line_and_column
      [method, VM::Atom.new(atom, filename: filename, line: line, column: column)]
    end

    rule(atom: simple(:atom)) do
      (line, column) = atom.line_and_column
      VM::Atom.new(atom, filename: filename, line: line, column: column)
    end

    rule(string: simple(:string)) do
      string.to_s
    end
class Parser
  class ParseError < StandardError
    attr_reader :line

    rule(quote: simple(:quote), sexp: subtree(:sexp)) do
      method = QUOTE_METHOD[quote.to_s]
      if sexp == '()'
        ['list']
      else
        [method] << sexp
      end
    def initialize(line)
      @line = line
    end

    rule(sexp: subtree(:sexp)) do
      if sexp == '()'
        []
      else
        sexp
      end
    def message
      "error parsing at line #{line}"
    end
  end
end

class Parser
  def initialize(code = nil, filename: nil)
    @code = code
    @filename = filename
    @parser = LISP::Parser.new
    @transform = LISP::Transform.new
  end

  def parse(code = @code)
    code.strip!
    return [] if code.empty?
    result = @parser.parse(code)
    @transform.apply(result, filename: @filename)
  def parse
    raise 'This method is implemented in Rust.'
  end
end

library = Fiddle::dlopen('target/release/libscheme_vm.dylib')
init_parser = Fiddle::Function.new(library['init_parser'], [], Fiddle::TYPE_VOID)
init_parser.call

M spec/parser_spec.rb => spec/parser_spec.rb +7 -7
@@ 1,9 1,9 @@
require_relative './spec_helper'

describe Parser do
fdescribe Parser do
  describe '#parse' do
    before do
      @result = subject.parse(<<-END)
    subject do
      described_class.new(<<-END)
        ; comment
        'foo
        '(1 2)


@@ 20,14 20,14 @@ describe Parser do
    end

    it 'parses s-expressions' do
      expect(@result).to eq([
      expect(subject.parse).to eq([
        ['quote', 'foo'],
        ['quote', ['1', '2']],
        ['list'],
        ['quote', []],
        ['unquote', 'foo'],
        ['unquote', ['foo', 'bar']], nil, nil,
        ['unquote', ['foo', 'bar']],
        ['print', 'space in identifier'],
        ['if', ['<', '1', '2'], nil,
        ['if', ['<', '1', '2'],
               'x',
               ['foo', ['bar', ['baz', '"this is a string"']]]]
      ])

M src/lib.rs => src/lib.rs +6 -2
@@ 1,11 1,13 @@
extern crate libc;
extern crate ruby_sys;
#[macro_use]
extern crate lazy_static;

#[macro_use] mod rb;
use rb::{CallbackPtr, Value, RB_NIL};

mod atom;
mod tests;
mod quotes;

mod lisp {
    include!(concat!(env!("OUT_DIR"), "/lisp.rs"));


@@ 17,8 19,10 @@ fn parse(rself: Value) -> Value {
        Ok(ast) => ast,
        Err(err) => {
            //let expected = rb::vec2rbarr(
            //    err.expected.iter().cloned().map(|e| rb::str_new(&e.to_string())).collect()
                //err.expected.iter().cloned().map(|e| rb::str_new(&e.to_string())).collect()
            //);
            println!("{}", err.column);
            println!("{:?}", err.expected);
            let c_parser = rb::const_get("Parser", &RB_NIL);
            let c_parse_error = rb::const_get("ParseError", &c_parser);
            let line = int2rbnum!(err.line);

M src/lisp.rustpeg => src/lisp.rustpeg +13 -3
@@ 1,6 1,7 @@
use rb;
use rb::Value;
use atom::atom;
use quotes::QUOTES;

whitespace
	= [ \t\n]+


@@ 9,10 10,19 @@ escape
	= "\\" .

string -> Option<Value>
	= "\"" s:$(escape / [^"])* "\"" { Some(rb::str_new(&s.to_string())) }
	= s:$("\"" (escape / [^"])* "\"") { Some(rb::str_new(&s.to_string())) }

delimited_identifier -> &'input str
	= "|" i:$([^|]+) "|" {i}

simple_atom -> Value
	= a:(delimited_identifier / $([^\(\) \t\n\[\]\{\}\|"]+)) { atom(&a) }

quoted_atom -> Value
	= q:quote a:simple_atom { rb::vec2rbarr(vec![q, a]) }

atom -> Option<Value>
	= n:$([^\(\) \t\n\[\]\{\}"]+) { Some(atom(&n)) }
	= a:(quoted_atom / simple_atom) { Some(a) }

sexp -> Option<Value>
	= n:(quoted_sexp / simple_sexp) { Some(n) }


@@ 24,7 34,7 @@ expression -> Option<Value>
	= string / comment / sexp / atom

quote -> Value
	= n:$("'" / ",@" / "," / "`") { rb::str_new(&n.to_string()) }
	= q:$("'" / ",@" / "," / "`") { rb::str_new(&QUOTES.get(&q).unwrap().to_string()) }

quoted_sexp -> Value
	= q:quote s:simple_sexp { rb::vec2rbarr(vec![q, s]) }

A src/quotes.rs => src/quotes.rs +12 -0
@@ 0,0 1,12 @@
use std::collections::HashMap;

lazy_static! {
    pub static ref QUOTES: HashMap<&'static str, &'static str> = {
        let mut q = HashMap::new();
        q.insert("'",  "quote");
        q.insert(",@", "unquote-splicing");
        q.insert(",",  "unquote");
        q.insert("`",  "quasiquote");
        q
    };
}

D src/tests.rs => src/tests.rs +0 -37
@@ 1,37 0,0 @@
#[cfg(test)]
mod tests {
    use lisp;
    use rb;
    use ruby_sys::vm::{ruby_init};
    use ruby_sys::fixnum::{rb_num2int};
    use ruby_sys::util::{rb_funcallv};

    #[test]
    fn program() {
        unsafe { ruby_init() };
        let program = lisp::program("
            ; comment
            'foo
            '(1 2)
            '()
            ,foo
            ,(foo bar) #; (baz) #;6
            #| this is a
               multi-line comment |#
            (print |space in identifier|)
            (if (< 1 2) #;(2 3)
                x ; another comment
                (foo (bar (baz \"this is a string\"))))
        ");
        match program {
            Ok(value) => {
                let args = rb::ary_new();
                let size = unsafe {
                    rb_num2int(rb_funcallv(value, str2rbid!("size"), 0, &args))
                };
                assert_eq!(7, size);
            },
            Err(_) => panic!()
        }
    }
}

D test.rb => test.rb +0 -33
@@ 1,33 0,0 @@
require 'fiddle'
require 'benchmark'
require_relative './vm'

class Parser
  class ParseError < StandardError
    attr_reader :line

    def initialize(line)
      @line = line
    end

    def message
      "error parsing at line #{line}"
    end
  end

  def initialize(code = nil, filename: nil)
    @code = code
    @filename = filename
  end

  def parse
    raise 'This method is implemented in Rust. If you see this error, then the dynamic library was not compiled/loaded.'
  end
end

library = Fiddle::dlopen('target/release/libscheme_vm.dylib')
init_parser = Fiddle::Function.new(library['init_parser'], [], Fiddle::TYPE_VOID)
init_parser.call

filename = ARGV.first
p Parser.new(File.read(filename), filename: filename).parse