~singpolyma/dhall-ruby

d83861ee4b0c538f419586d4869f570de890ff38 — Stephen Paul Weber 2 years ago f9b8510
First pass at import resolution
M Gemfile => Gemfile +1 -0
@@ 4,3 4,4 @@ source "https://rubygems.org"

gem "cbor"
gem "value_semantics"
gem "promise.rb"

M lib/dhall.rb => lib/dhall.rb +1 -0
@@ 4,3 4,4 @@ require "dhall/ast"
require "dhall/builtins"
require "dhall/binary"
require "dhall/normalize"
require "dhall/resolve"

M lib/dhall/ast.rb => lib/dhall/ast.rb +113 -13
@@ 550,36 550,136 @@ module Dhall
		end

		class URI
			include(ValueSemantics.for_attributes do
				headers   Either(nil, Expression)
				authority ::String
				path      ArrayOf(::String)
				query     Either(nil, ::String)
				fragment  Either(nil, ::String)
			end)

			def initialize(headers, authority, *path, query, fragment)
				@headers = headers
				@authority = authority
				@path = path
				@query = query
				@fragment = fragment
				super(
					headers:   headers,
					authority: authority,
					path:      path,
					query:     query,
					fragment:  fragment
				)
			end

			def self.from_uri(uri)
				(uri.scheme == "https" ? Https : Http).new(
					nil,
					"#{uri.host}:#{uri.port}",
					uri.path.split(/\//)[1..-1],
					uri.query,
					nil
				)
			end

			def uri
				URI("#{scheme}://#{authority}/#{path.join("/")}?#{query}")
			end
		end

		class Http < URI
			def resolve(resolver)
				resolver.resolve_http(self)
			end

			def scheme
				"http"
			end
		end

		class Http < URI; end
		class Https < URI; end
		class Https < URI
			def resolve(resolver)
				resolver.resolve_https(self)
			end

			def scheme
				"https"
			end
		end

		class Path
			include(ValueSemantics.for_attributes do
				path ArrayOf(::String)
			end)

			def initialize(*path)
				@path = path
				super(path: path)
			end

			def self.from_string(s)
				parts = s.split(/\//)
				if parts.first == ""
					AbsolutePath.new(*parts[1..-1])
				elsif parts.first == "~"
					RelativeToHomePath.new(*parts[1..-1])
				else
					RelativePath.new(*parts)
				end
			end

			def resolve(resolver)
				resolver.resolve_path(self)
			end

			def to_s
				pathname.to_s
			end
		end

		class AbsolutePath < Path
			def pathname
				Pathname.new("/").join(*path)
			end
		end

		class RelativePath < Path
			def pathname
				Pathname.new(".").join(*path)
			end
		end

		class RelativeToParentPath < Path
			def pathname
				Pathname.new("..").join(*path)
			end
		end

		class AbsolutePath < Path; end
		class RelativePath < Path; end
		class RelativeToParentPath < Path; end
		class RelativeToHomePath < Path; end
		class RelativeToHomePath < Path
			def pathname
				Pathname.new("~").join(*@path)
			end
		end

		class EnvironmentVariable
			def initialize(var)
				@var = var
			end

			def value
				ENV.fetch(@var)
			end

			def resolve(resolver)
				val = ENV.fetch(@var)
				if val =~ /\Ahttps?:\/\//
					URI.from_uri(URI(value))
				else
					Path.from_string(val)
				end.resolve(resolver)
			end
		end

		class MissingImport; end
		class MissingImport
			def resolve(*)
				Promise.new.reject(ImportFailedException.new("missing"))
			end
		end

		class IntegrityCheck
			def initialize(protocol, data)

M lib/dhall/binary.rb => lib/dhall/binary.rb +7 -2
@@ 176,7 176,11 @@ module Dhall
	end

	class Import
		IMPORT_TYPES = [Expression, Text].freeze
		IMPORT_TYPES = [
			Dhall.method(:from_binary),
			->(x) { Text.new(value: x) }
		].freeze

		PATH_TYPES = [
			Http, Https,
			AbsolutePath, RelativePath, RelativeToParentPath, RelativeToHomePath,


@@ 184,7 188,8 @@ module Dhall
		].freeze

		def self.decode(integrity_check, import_type, path_type, *parts)
			parts[0] = Dhall.decode(parts[0]) if path_type && !parts[0].nil?
			parts[0] = Dhall.decode(parts[0]) if path_type < 2 && !parts[0].nil?

			new(
				integrity_check.nil? ? nil : IntegrityCheck.new(*integrity_check),
				IMPORT_TYPES[import_type],

A lib/dhall/resolve.rb => lib/dhall/resolve.rb +214 -0
@@ 0,0 1,214 @@
# frozen_string_literal: true

require "set"
require "promise.rb"

require "dhall/ast"
require "dhall/util"

module Dhall
	class ImportFailedException < StandardError; end
	class ImportBannedException < ImportFailedException; end
	class ImportLoopException < ImportBannedException; end

	module Resolvers
		ReadPathSources = lambda do |sources|
			sources.map do |source|
				Promise.resolve(nil).then { source.pathname.read }
			end
		end

		ReadHttpSources = lambda do |sources|
			sources.map do |source|
				Promise.resolve(nil).then do
					Net::HTTP.get(source.uri)
				end
			end
		end

		RejectSources = lambda do |sources|
			sources.map do |source|
				Promise.new.reject(ImportBannedException.new(source))
			end
		end

		class ResolutionSet
			attr_reader :reader

			def initialize(reader)
				@reader = reader
				@parents = Set.new
				@set = Hash.new { |h, k| h[k] = [] }
			end

			def register(source)
				p = Promise.new
				if @parents.include?(source)
					p.reject(ImportLoopException.new(source))
				else
					@set[source] << p
				end
				p
			end

			def resolutions
				sources, promises = @set.to_a.transpose
				[Array(sources), Array(promises)]
			end

			def child(parent_source)
				dup.tap do |c|
					c.instance_eval do
						@parents = @parents.dup + [parent_source]
						@set = Hash.new { |h, k| h[k] = [] }
					end
				end
			end
		end

		class Default
			def initialize(
				path_reader: ReadPathSources,
				http_reader: ReadHttpSources,
				https_reader: http_reader
			)
				@path_resolutions = ResolutionSet.new(path_reader)
				@http_resolutions = ResolutionSet.new(http_reader)
				@https_resolutions = ResolutionSet.new(https_reader)
			end

			def resolve_path(path_source)
				@path_resolutions.register(path_source)
			end

			def resolve_http(http_source)
				@http_resolutions.register(http_source)
			end

			def resolve_https(https_source)
				@https_resolutions.register(https_source)
			end

			def finish!
				[
					@path_resolutions,
					@http_resolutions,
					@https_resolutions
				].each do |rset|
					Util.match_result_promises(*rset.resolutions, &rset.reader)
				end
				freeze
			end

			def child(parent_source)
				dup.tap do |c|
					c.instance_eval do
						@path_resolutions = @path_resolutions.child(parent_source)
						@http_resolutions = @http_resolutions.child(parent_source)
						@https_resolutions = @https_resolutions.child(parent_source)
					end
				end
			end
		end

		class LocalOnly < Default
			def initialize(path_reader: ReadPathSources)
				super(
					path_reader:  path_reader,
					http_reader:  RejectSources,
					https_reader: RejectSources
				)
			end
		end

		class None < Default
			def initialize
				super(
					path_reader:  RejectSources,
					http_reader:  RejectSources,
					https_reader: RejectSources
				)
			end
		end
	end

	class ExpressionResolver
		@@registry = {}

		def self.for(expr)
			@@registry.find { |k, _| k === expr }.last.new(expr)
		end

		def self.register_for(kase)
			@@registry[kase] = self
		end

		def initialize(expr)
			@expr = expr
		end

		def resolve(resolver)
			Util.promise_all_hash(
				@expr.to_h.each_with_object({}) { |(attr, value), h|
					h[attr] = ExpressionResolver.for(value).resolve(resolver)
				}
			).then { |h| @expr.with(h) }
		end

		class ImportResolver < ExpressionResolver
			register_for Import

			def resolve(resolver)
				@expr.instance_eval do
					@path.resolve(resolver).then do |expr|
						@import_type.call(expr).resolve(resolver.child(@path))
					end
				end
			end
		end

		class FallbackResolver < ExpressionResolver
			register_for Operator::ImportFallback

			def resolve(resolver)
				ExpressionResolver.for(@expr.lhs).resolve(resolver).catch do
					ExpressionResolver.for(@expr.rhs).resolve(resolver)
				end
			end
		end

		class ArrayResolver < ExpressionResolver
			def resolve(resolver)
				Promise.all(
					@expr.map { |e| ExpressionResolver.for(e).resolve(resolver) }
				)
			end
		end

		class HashResolver < ExpressionResolver
			def resolve(resolver)
				Util.promise_all_hash(Hash[@expr.map do |k, v|
					[k, ExpressionResolver.for(v).resolve(resolver)]
				end])
			end
		end

		register_for Expression

		class IdentityResolver < ExpressionResolver
			register_for Object

			def resolve(*)
				@expr
			end
		end
	end

	class Expression
		def resolve(resolver=Resolvers::Default.new)
			p = ExpressionResolver.for(self).resolve(resolver)
			resolver.finish!
			p
		end
	end
end

M lib/dhall/util.rb => lib/dhall/util.rb +24 -0
@@ 34,5 34,29 @@ module Dhall
					other.size >= @min && other.size <= @max
			end
		end

		def self.match_results(xs=nil, ys=nil)
			Array(xs).each_with_index.map do |r, idx|
				yield r, ys[idx]
			end
		end

		def self.match_result_promises(xs=nil, ys=nil)
			match_results(yield(Array(xs)), ys) do |promise, promises|
				promises.each { |p| p.fulfill(promise) }
			end
		end

		def self.promise_all_hash(hash)
			keys, promises = hash.to_a.transpose

			return Promise.resolve(hash) unless keys

			Promise.all(promises).then do |values|
				Hash[Util.match_results(keys, values) do |k, v|
					[k, v]
				end]
			end
		end
	end
end

A test/test_resolve.rb => test/test_resolve.rb +190 -0
@@ 0,0 1,190 @@
# frozen_string_literal: true

require "base64"
require "minitest/autorun"

require "dhall/resolve"
require "dhall/normalize"
require "dhall/binary"

DIRPATH = Pathname.new(File.dirname(__FILE__))
TESTS = DIRPATH + "normalization/"

class TestResolve < Minitest::Test
	def setup
		@resolver = Dhall::Resolvers::Default.new(
			path_reader: lambda do |sources|
				sources.map do |source|
					Promise.resolve(Base64.decode64({
						"var"      => "AA",
						"import"   => "hRgY9gADY3Zhcg",
						"a"        => "hRgY9gADYWI",
						"b"        => "hRgY9gADYWE",
						"self"     => "hRgY9gADZHNlbGY",
						"text"     => "aGFp",
						"moretext" => "hRgY9gEDZHRleHQ",
						"2text"    => "hAMGhRgY9gEDZHRleHSFGBj2AANobW9yZXRleHQ"
					}.fetch(source.pathname.to_s)))
				end
			end
		)
	end

	def test_nothing_to_resolve
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Variable["_"]
		)

		assert_equal expr, expr.resolve(Dhall::Resolvers::None.new).sync
	end

	def test_import_as_text
		expr = Dhall::Import.new(
			nil,
			->(x) { Dhall::Text.new(value: x) },
			Dhall::Import::RelativePath.new("text")
		)

		assert_equal(
			Dhall::Text.new(value: "hai"),
			expr.resolve(@resolver).sync
		)
	end

	def test_one_level_to_resolve
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::RelativePath.new("var")
			)
		)

		assert_equal(
			expr.with(body: Dhall::Variable["_"]),
			expr.resolve(@resolver).sync
		)
	end

	def test_two_levels_to_resolve
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::RelativePath.new("import")
			)
		)

		assert_equal(
			expr.with(body: Dhall::Variable["_"]),
			expr.resolve(@resolver).sync
		)
	end

	def test_self_loop
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::RelativePath.new("self")
			)
		)

		assert_raises Dhall::ImportLoopException do
			expr.resolve(@resolver).sync
		end
	end

	def test_two_level_loop
		expr = Dhall::Function.of_arguments(
			Dhall::Variable["Natural"],
			body: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::RelativePath.new("a")
			)
		)

		assert_raises Dhall::ImportLoopException do
			expr.resolve(@resolver).sync
		end
	end

	def test_two_references_no_loop
		expr = Dhall::Import.new(
			nil,
			Dhall.method(:from_binary),
			Dhall::Import::RelativePath.new("2text")
		)

		assert_equal(
			Dhall::Operator::TextConcatenate.new(
				lhs: Dhall::Text.new(value: "hai"),
				rhs: Dhall::Text.new(value: "hai")
			),
			expr.resolve(@resolver).sync
		)
	end

	def test_missing
		expr = Dhall::Import.new(
			nil,
			Dhall.method(:from_binary),
			Dhall::Import::MissingImport.new
		)

		assert_raises Dhall::ImportFailedException do
			expr.resolve(@resolver).sync
		end
	end

	def test_fallback_to_expr
		expr = Dhall::Operator::ImportFallback.new(
			lhs: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::MissingImport.new
			),
			rhs: Dhall::Variable["fallback"]
		)

		assert_equal Dhall::Variable["fallback"], expr.resolve(@resolver).sync
	end

	def test_fallback_to_import
		expr = Dhall::Operator::ImportFallback.new(
			lhs: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::MissingImport.new
			),
			rhs: Dhall::Import.new(
				nil,
				Dhall.method(:from_binary),
				Dhall::Import::RelativePath.new("import")
			)
		)

		assert_equal Dhall::Variable["_"], expr.resolve(@resolver).sync
	end

	# Sanity check that all expressions can pass through the resolver
	Pathname.glob(TESTS + "**/*A.dhallb").each do |path|
		test = path.relative_path_from(TESTS).to_s.sub(/A\.dhallb$/, "")
		next if test =~ /prelude\//
		next if test =~ /remoteSystems/

		define_method("test_#{test.gsub(/\//, "_")}") do
			Dhall::Function.disable_alpha_normalization! if test =~ /^standard\//
			assert_equal(
				Dhall.from_binary(TESTS + "#{test}B.dhallb"),
				Dhall.from_binary(path.read).resolve.sync.normalize
			)
			Dhall::Function.enable_alpha_normalization! if test =~ /^standard\//
		end
	end
end

A test/test_resolvers.rb => test/test_resolvers.rb +90 -0
@@ 0,0 1,90 @@
# frozen_string_literal: true

require "minitest/autorun"

require "dhall/resolve"

class TestResolvers < Minitest::Test
	def test_default_resolver_path
		resolver = Dhall::Resolvers::Default.new(
			path_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
			end
		)
		source = Dhall::Import::AbsolutePath.new("dhall", "common", "x.dhall")
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
	end

	def test_default_resolver_path_from_env
		ENV["__DHALL_IMPORT_TEST"] = "/dhall/common/x.dhall"
		resolver = Dhall::Resolvers::Default.new(
			path_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
			end
		)
		source = Dhall::Import::EnvironmentVariable.new("__DHALL_IMPORT_TEST")
		promise = source.resolve(resolver)
		resolver.finish!

		expected = Dhall::Import::AbsolutePath.new("dhall", "common", "x.dhall")
		assert_equal expected, promise.sync
	end

	def test_default_resolver_http
		resolver = Dhall::Resolvers::Default.new(
			http_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
			end
		)
		source = Dhall::Import::Http.new(nil, "example.com", "x.dhall", nil, nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
	end

	def test_default_resolver_https
		resolver = Dhall::Resolvers::Default.new(
			https_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
			end
		)
		source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil, nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
	end

	def test_default_resolver_https_uses_http
		resolver = Dhall::Resolvers::Default.new(
			http_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
			end
		)
		source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil, nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
	end

	def test_local_only_resolver_rejects_http
		resolver = Dhall::Resolvers::LocalOnly.new
		source = Dhall::Import::Http.new(nil, "example.com", "x.dhall", nil, nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_raises Dhall::ImportBannedException do
			promise.sync
		end
	end

	def test_none_resolver_rejects_local
		resolver = Dhall::Resolvers::None.new
		source = Dhall::Import::AbsolutePath.new("dhall", "common", "x.dhall")
		promise = source.resolve(resolver)
		resolver.finish!
		assert_raises Dhall::ImportBannedException do
			promise.sync
		end
	end
end