~tim/scheme-vm

772568ff8c4d05d0075648e278fbb383234b0956 — Tim Morgan 5 years ago beb5107
Add beginning of library support; add proper closures
M compiler.rb => compiler.rb +33 -7
@@ 11,6 11,7 @@ class Compiler
    @filename = filename
    @arguments = arguments
    @syntax = {}              # macro transformers
    @libs = {}                # loaded libraries
    @mangled_identifiers = {} # used for macro hygiene
    @source = {}              # store source code for each file compiled
    @source[filename] = code


@@ 19,13 20,11 @@ class Compiler
    @sexps += Parser.new(code, filename: filename).parse if code
  end

  attr_reader :variables, :filename, :arguments, :syntax, :source
  attr_reader :variables, :filename, :arguments, :syntax, :source, :libs

  def compile(code = nil, keep_last: false, halt: true)
    @sexps += Parser.new(code, filename: filename).parse if code
    instructions = compile_sexps(@sexps, { syntax: @syntax }, filename: filename, keep_last: keep_last)
    instructions << VM::HALT if halt
    optimize(instructions)
    compile_sexps(@sexps, options: { syntax: @syntax }, halt: halt, keep_last: keep_last)
  end

  def include_code(paths)


@@ 76,13 75,15 @@ class Compiler

  private

  def compile_sexps(sexps, options = {}, filename:, keep_last: false)
  def compile_sexps(sexps, options: {}, halt: false, keep_last: false)
    options[:locals] ||= {}
    sexps
    instructions = sexps
      .each_with_index
      .map { |s, i| compile_sexp(s, options.merge(use: i == sexps.size - 1 && keep_last)) }
      .flatten
      .compact
    instructions << VM::HALT if halt
    optimize(instructions)
  end

  def optimize(instructions)


@@ 102,6 103,8 @@ class Compiler
      call(sexp, options)
    elsif name == 'include'
      do_include(args, name.filename, options)
    elsif name == 'import'
      do_import(args, name.filename, options)
    elsif (built_in_name = built_in_function_name(name))
      send(built_in_name, args, options)
    elsif (macro = find_syntax(name, options))


@@ 492,10 495,33 @@ class Compiler
      fail "include expects a string, but got #{path.inspect}" unless path =~ /\A"(.+)?"\z/
      filename = "#{$1}.scm"
      sexps = parse_file(filename, relative_to: relative_to)
      compile_sexps(sexps, { syntax: options[:syntax], locals: options[:locals] }, filename: filename)
      compile_sexps(sexps, options: { syntax: options[:syntax], locals: options[:locals] })
    end
  end

  def do_import((*sets), relative_to, options)
    sets.map do |set|
      name = set.join('/')
      path = '../' + name # FIXME: remove the ../
      [
        do_include(["\"#{path}\""], relative_to, options),
        @libs[name].map do |export|
          [VM::IMPORT_LIB, name, export]
        end
      ]
    end
  end

  def do_define_library((name, *declarations), options)
    @libs[name.join('/')] = declarations.flat_map { |(type, *args)| args if type == 'export' }.compact
    begins = declarations.flat_map { |(type, *body)| body if type == 'begin' }.compact
    [
      VM::SET_LIB, name.join('/'),
      begins.map { |s| compile_sexp(s, options) },
      VM::ENDL
    ]
  end

  def do_write(args, options)
    [
      args.map { |arg| compile_sexp(arg, options.merge(use: true)) },

M spec/compiler_spec.rb => spec/compiler_spec.rb +27 -0
@@ 1194,5 1194,32 @@ describe Compiler do
        ])
      end
    end

    context 'define-library' do
      before do
        @result = subject.compile(<<-END)
          (define-library (my-lib 1)
            (begin
              (define foo "foo"))
            (export foo))
        END
      end

      it 'records export names for the library' do
        expect(subject.libs).to eq({
          'my-lib/1' => ['foo']
        })
      end

      it 'compiles into vm instructions' do
        expect(d(@result)).to eq([
          'VM::SET_LIB', 'my-lib/1',
          'VM::PUSH_STR', 'foo',
          'VM::SET_LOCAL', 'foo',
          'VM::ENDL',
          'VM::HALT'
        ])
      end
    end
  end
end

A spec/fixtures/library-test.scm => spec/fixtures/library-test.scm +5 -0
@@ 0,0 1,5 @@
(define-library (fixtures library-test)
  (export foo)
  (begin
    (define (foo)
      12)))

A spec/lib/import-spec.scm => spec/lib/import-spec.scm +5 -0
@@ 0,0 1,5 @@
(include "assert")

(import (fixtures library-test))

(assert (eq? 12 (foo)))

M spec/lib_spec.rb => spec/lib_spec.rb +5 -2
@@ 3,10 3,13 @@ require 'stringio'

Dir[File.expand_path('../lib/**/*.scm', __FILE__)].each do |path|
  describe File.split(path).last do
    it 'passes all tests' do
    code = File.read(path)
    focus = !(code =~ /^; focus/).nil?
    skip = !(code =~ /^; skip/).nil?
    it 'passes all tests', focus: focus, skip: skip do
      failed = false
      out = StringIO.new
      Program.new(File.read(path), filename: File.split(path).last, stdout: out).run
      Program.new(code, filename: path, stdout: out).run
      out.rewind
      result = out.read
      if result != ''

M spec/vm_spec.rb => spec/vm_spec.rb +86 -1
@@ 382,7 382,7 @@ describe VM do
  end

  describe 'PUSH_REMOTE' do
    context 'pushing a local defined prior to this function (closure)' do
    context 'pushing a local defined prior to this function' do
      before do
        subject.execute([
          VM::PUSH_NUM, 10,


@@ 406,6 406,36 @@ describe VM do
        expect(stdout.read).to eq('10')
      end
    end

    context 'referencing a variable in lexical scope but not in dynamic scope (closure)' do
      before do
        subject.execute([
          VM::PUSH_FUNC,

          VM::PUSH_NUM, '10',
          VM::SET_LOCAL, 'x',

          VM::PUSH_FUNC,
          VM::PUSH_REMOTE, 'x',
          VM::RETURN,
          VM::ENDF,

          VM::RETURN,
          VM::ENDF,

          VM::CALL, # call the outer function
          VM::CALL, # call the inner function

          VM::HALT
        ])
      end

      it 'captures the variable' do
        expect(subject.stack_values).to eq([
          VM::Int.new(10)
        ])
      end
    end
  end

  describe 'SET_ARGS and PUSH_ARG and PUSH_ARGS' do


@@ 1108,4 1138,59 @@ describe VM do
      expect(subject.pop_val).to eq(VM::Int.new(0))
    end
  end

  describe 'SET_LIB and IMPORT_LIB' do
    before do
      subject.execute([
        VM::SET_LIB, 'my-lib',

        VM::PUSH_STR, 'foo',
        VM::SET_LOCAL, 'foo',

        VM::PUSH_STR, 'bar',
        VM::SET_LOCAL, 'private',

        VM::PUSH_FUNC,
        VM::PUSH_REMOTE, 'private',
        VM::RETURN,
        VM::ENDF,
        VM::SET_LOCAL, 'bar-fn',

        VM::ENDL,
        VM::HALT # pause here
      ])
    end

    it 'stores the library context and locals defined within' do
      expect(subject.libs['my-lib']).to be
      expect(subject.libs['my-lib'][:locals].keys).to eq(['foo', 'private', 'bar-fn'])
    end

    it 'imports the named binding into the local frame' do
      expect(subject.locals).to eq({})
      subject.execute([
        VM::IMPORT_LIB, 'my-lib', 'foo',
        VM::PUSH_LOCAL, 'foo',
        VM::HALT
      ])
      expect(subject.stack_values).to eq([
        VM::ByteArray.new('foo')
      ])
      expect(subject.locals.keys).to eq(['foo'])
    end

    it 'allows an imported binding to reference a non-imported one' do
      expect(subject.locals).to eq({})
      subject.execute([
        VM::IMPORT_LIB, 'my-lib', 'bar-fn',
        VM::PUSH_LOCAL, 'bar-fn',
        VM::CALL,
        VM::HALT
      ])
      expect(subject.stack_values).to eq([
        VM::ByteArray.new('bar')
      ])
      expect(subject.locals.keys).to eq(['bar-fn'])
    end
  end
end

M todo.md => todo.md +1 -0
@@ 3,6 3,7 @@
- [x] GC
- [x] show proper error when source is another file
- [x] optimize `JUMP` to `RETURN` if jump lands on a return (fixes TCE in some cases)
- [x] proper closures

## Missing things from R7RS:


M vm.rb => vm.rb +12 -5
@@ 39,6 39,9 @@ class VM
    ['PUSH_ARG',      0],
    ['PUSH_ARGS',     0],
    ['PUSH_FUNC',     0],
    ['ENDF',          0],
    ['PUSH_LIB',      0],
    ['ENDL',          0],
    ['STR_REF',       0],
    ['STR_LEN',       0],
    ['LIST_TO_STR',   0],


@@ 55,18 58,19 @@ class VM
    ['CMP_EQ_NUM',    0],
    ['CMP_NULL',      0],
    ['DUP',           0],
    ['ENDF',          0],
    ['INT',           1],
    ['JUMP',          1],
    ['JUMP_IF_FALSE', 1],
    ['CALL',          0],
    ['APPLY',         0],
    ['RETURN',        0],
    ['SET_LIB',       1],
    ['SET_LOCAL',     1],
    ['SET_REMOTE',    1],
    ['SET_ARGS',      0],
    ['SET_CAR',       0],
    ['SET_CDR',       0],
    ['IMPORT_LIB',    2],
    ['HALT',          0],
    ['DEBUG',         0]
  ]


@@ 87,15 91,17 @@ class VM
    VM::Pair
  ]

  attr_reader :stack, :heap, :stdout, :ip, :call_stack, :call_args
  attr_reader :stack, :heap, :stdout, :ip, :call_stack, :closures, :call_args, :libs

  def initialize(instructions = [], args: [], stdout: $stdout)
    @ip = 0              # instruction pointer
    @stack = []          # operand stack
    @call_stack = []     # call frame stack
    @call_stack << { locals: {}, args: args }
    @libs = {}           # library definitions
    @heap = []           # a heap "address" is an index into this array
    @call_args = []      # used for next CALL
    @closures = {}       # store function ip and locals available
    @stdout = stdout
    @executable = []     # ranges of executable heap (w^x)
    load_code(instructions)


@@ 258,12 264,13 @@ class VM
    stdout.print(val.to_s)
  end

  def fetch_func_body
  def fetch_until(end_instruction)
    body = []
    while (instruction = fetch) != ENDF
    while (instruction = fetch) != end_instruction
      (_name, arity) = INSTRUCTIONS[instruction]
      body << instruction
      body << fetch_func_body + [ENDF] if instruction == PUSH_FUNC
      body << fetch_until(ENDF) + [ENDF] if instruction == PUSH_FUNC
      body << fetch_until(ENDL) + [ENDL] if instruction == PUSH_LIB
      arity.times { body << fetch } # skip args
    end
    body.flatten

M vm/gc.rb => vm/gc.rb +5 -1
@@ 18,7 18,7 @@ class VM

    def active?(candidate)
      return true if singleton?(candidate)
      (call_stack_locations + op_stack_locations).uniq.each do |location|
      (closure_locations + call_stack_locations + op_stack_locations).uniq.each do |location|
        return true if location == candidate
        value = @vm.heap[location]
        return true if value.is_a?(VM::Pair) && active_in_pair?(candidate, value)


@@ 26,6 26,10 @@ class VM
      false
    end

    def closure_locations
      @vm.closures.values.flat_map { |c| c[:locals].values }.uniq
    end

    def call_stack_locations
      @vm.call_stack.flat_map { |f| f[:locals].values }.uniq
    end

M vm/operations.rb => vm/operations.rb +35 -9
@@ 58,15 58,21 @@ class VM
    def do_push_local
      name = fetch
      address = locals[name]
      fail VariableUndefined.new(name) unless address
      fail VariableUndefined, name unless address
      push(address)
    end

    def do_push_remote
      name = fetch
      frame_locals = @call_stack.reverse.lazy.map { |f| f[:locals] }.detect { |l| l[name] }
      fail VariableUndefined.new(name) unless frame_locals
      address = frame_locals.fetch(name)
      func = @call_stack.last[:func]
      closure_locals = func ? @closures[func][:locals] : {}
      if closure_locals.key?(name)
        address = closure_locals.fetch(name)
      else
        frame_locals = @call_stack.reverse.lazy.map { |f| f[:locals] }.detect { |l| l.key?(name) }
        fail VariableUndefined, name unless frame_locals
        address = frame_locals.fetch(name)
      end
      push(address)
    end



@@ 82,7 88,24 @@ class VM

    def do_push_func
      push(@ip)
      fetch_func_body # discard
      @closures[@ip] = { locals: locals }
      fetch_until(ENDF) # discard
    end

    def do_set_lib
      name = fetch
      @call_stack << { lib: name, locals: {} }
    end

    def do_endl
      frame = @call_stack.pop
      @libs[frame[:lib]] = { locals: frame[:locals] }
    end

    def do_import_lib
      lib = fetch
      binding = fetch
      locals[binding] = @libs[lib][:locals][binding]
    end

    def do_str_ref


@@ 103,13 126,15 @@ class VM
    end

    def do_call
      new_ip = pop
      if @heap[@ip] == RETURN
        @call_stack.last[:func] = new_ip
        @call_stack.last[:args] = @call_args
      else
        @call_stack << { return: @ip, locals: {}, args: @call_args }
        @call_stack << { func: new_ip, return: @ip, locals: {}, args: @call_args }
      end
      fail CallStackTooDeep, 'call stack too deep' if @call_stack.size > MAX_CALL_DEPTH
      @ip = pop
      @ip = new_ip
    end

    def do_apply


@@ 118,9 143,10 @@ class VM
        @call_args.push(pair.address)
        pair = @heap[pair.next_node]
      end
      @call_stack << { return: @ip, locals: {}, args: @call_args }
      new_ip = pop
      @call_stack << { func: new_ip, return: @ip, locals: {}, args: @call_args }
      fail CallStackTooDeep, 'call stack too deep' if @call_stack.size > MAX_CALL_DEPTH
      @ip = pop
      @ip = new_ip
    end

    def do_return(debug)