~singpolyma/dhall-ruby

59c4dfe5385629177ad61bd8edf671d4398df0ee — Stephen Paul Weber 5 years ago df3138f
Timeout mechanism

Sets timeouts in HTTP readers, checks for deadline exceeded before each
resolution step, and uses Timeout::timeout for pure-ruby computations.
M README.md => README.md +8 -0
@@ 47,6 47,14 @@ Wherever possible, you should use the `Promise` API and treat `Dhall.load` as an

**This will block the thread it is run from until the whole load operation is complete.  Never call `#sync` from an async context.**

### Timeout

It is possible for malicious entities to craft Dhall expressions which take an unreasonable amount of time to load.  To protect against this, `Dhall.load` implements a timeout mechanism with a default of 10 seconds.  You may specify an alternate timeout like so:

	Dhall.load("1 + 1", timeout: 1)               # 1 second timeout
	Dhall.load("1 + 1", timeout: 0.1)             # 0.1 second timeut
	Dhall.load("1 + 1", timeout: Float::INFINITY) # Never timeout

### Customizing Import Resolution

You may optionally pass `Dhall.load` a resolver that will be used to resolve all imports during the load process:

M lib/dhall.rb => lib/dhall.rb +21 -8
@@ 13,25 13,38 @@ require "dhall/typecheck"
module Dhall
	using Dhall::AsDhall

	def self.load(source, resolver: Resolvers::Default.new)
	def self.load(
		source,
		resolver: Resolvers::Default.new,
		timeout: 10
	)
		deadline = Util::Deadline.for_timeout(timeout)
		Promise.resolve(nil).then {
			load_raw(source.to_s).resolve(resolver: resolver)
			load_raw(source.to_s, timeout: timeout).resolve(
				resolver: resolver.with_deadline(deadline)
			)
		}.then do |resolved|
			TypeChecker.for(resolved).annotate(TypeChecker::Context.new).normalize
			deadline.timeout_block do
				TypeChecker.for(resolved).annotate(TypeChecker::Context.new).normalize
			end
		end
	end

	def self.load_raw(source)
	def self.load_raw(source, timeout: 10)
		source = Util.text_or_binary(source)

		if source.encoding == Encoding::BINARY
			from_binary(source)
		else
			Parser.parse(source).value
		Util::Deadline.for_timeout(timeout).timeout_block do
			if source.encoding == Encoding::BINARY
				from_binary(source)
			else
				Parser.parse(source).value
			end
		end
	end

	def self.dump(o)
		CBOR.encode(o.as_dhall)
	end

	class TimeoutException < StandardError; end
end

M lib/dhall/ast.rb => lib/dhall/ast.rb +5 -5
@@ 1444,15 1444,15 @@ module Dhall
		end

		class Expression
			def self.call(import_value)
			def self.call(import_value, deadline: Util::NoDeadline.new)
				return import_value if import_value.is_a?(Dhall::Expression)

				Dhall.load_raw(import_value)
				Dhall.load_raw(import_value, timeout: deadline.timeout)
			end
		end

		class Text
			def self.call(import_value)
			def self.call(import_value, deadline: Util::NoDeadline.new)
				Dhall::Text.new(value: import_value)
			end
		end


@@ 1494,8 1494,8 @@ module Dhall
			path.chain_onto(relative_to).canonical
		end

		def parse_and_check(raw)
			integrity_check.check(import_type.call(raw))
		def parse_and_check(raw, deadline: Util::NoDeadline.new)
			integrity_check.check(import_type.call(raw, deadline: deadline))
		end

		def cache_key(relative_to)

M lib/dhall/resolve.rb => lib/dhall/resolve.rb +53 -9
@@ 31,6 31,7 @@ module Dhall
		end

		PreflightCORS = lambda do |source, parent_origin|
			timeout = source.deadline.timeout
			uri = source.uri
			if parent_origin != "localhost" && parent_origin != source.origin
				req = Net::HTTP::Options.new(uri)


@@ 41,7 42,11 @@ module Dhall
				r = Net::HTTP.start(
					uri.hostname,
					uri.port,
					use_ssl: uri.scheme == "https"
					use_ssl:       uri.scheme == "https",
					open_timeout:  timeout,
					ssl_timeout:   timeout,
					read_timeout:  timeout,
					write_timeout: timeout
				) { |http| http.request(req) }

				raise ImportFailedException, source if r.code != "200"


@@ 56,6 61,7 @@ module Dhall
			sources.map do |source|
				Promise.resolve(nil).then do
					PreflightCORS.call(source, parent_origin)
					timeout = source.deadline.timeout
					uri = source.uri
					req = Net::HTTP::Get.new(uri)
					source.headers.each do |header|


@@ 64,7 70,11 @@ module Dhall
					r = Net::HTTP.start(
						uri.hostname,
						uri.port,
						use_ssl: uri.scheme == "https"
						use_ssl:       uri.scheme == "https",
						open_timeout:  timeout,
						ssl_timeout:   timeout,
						read_timeout:  timeout,
						write_timeout: timeout
					) { |http| http.request(req) }

					raise ImportFailedException, source if r.code != "200"


@@ 111,7 121,7 @@ module Dhall
			def call(sources)
				@path_reader.call(sources).map.with_index do |promise, idx|
					source = sources[idx]
					if source.is_a?(Import::AbsolutePath) &&
					if source.canonical.is_a?(Import::AbsolutePath) &&
					   ["ipfs", "ipns"].include?(source.path.first)
						gateway_fallback(source, promise)
					else


@@ 148,7 158,7 @@ module Dhall

			def register(source)
				p = Promise.new
				if @parents.include?(source)
				if @parents.include?(source.canonical)
					p.reject(ImportLoopException.new(source))
				else
					@set[source] << p


@@ 163,6 173,8 @@ module Dhall

			def reader
				lambda do |sources|
					raise TimeoutException if sources.any? { |s| s.deadline.exceeded? }

					if @reader.arity == 2
						@reader.call(sources, @parents.last&.origin || "localhost")
					else


@@ 181,7 193,24 @@ module Dhall
			end
		end

		class SourceWithDeadline < SimpleDelegator
			attr_reader :deadline

			def initialize(source, deadline)
				@source = source
				@deadline = deadline

				super(source)
			end

			def to_uri(*args)
				self.class.new(super, deadline)
			end
		end

		class Standard
			attr_reader :deadline

			def initialize(
				path_reader: ReadPathSources,
				http_reader: StandardReadHttpSources,


@@ 192,9 221,18 @@ module Dhall
				@http_resolutions = ResolutionSet.new(http_reader)
				@https_resolutions = ResolutionSet.new(https_reader)
				@env_resolutions = ResolutionSet.new(environment_reader)
				@deadline = Util::NoDeadline.new
				@cache = {}
			end

			def with_deadline(deadline)
				dup.tap do |c|
					c.instance_eval do
						@deadline = deadline
					end
				end
			end

			def cache_fetch(key, &fallback)
				@cache.fetch(key) do
					Promise.resolve(nil).then(&fallback).then do |result|


@@ 204,11 242,15 @@ module Dhall
			end

			def resolve_path(path_source)
				@path_resolutions.register(path_source)
				@path_resolutions.register(
					SourceWithDeadline.new(path_source, @deadline)
				)
			end

			def resolve_environment(env_source)
				@env_resolutions.register(env_source)
				@env_resolutions.register(
					SourceWithDeadline.new(env_source, @deadline)
				)
			end

			def resolve_http(http_source)


@@ 216,8 258,9 @@ module Dhall
					resolver:    self,
					relative_to: Dhall::Import::RelativePath.new
				).then do |headers|
					source = http_source.with(headers: headers.normalize)
					@http_resolutions.register(
						http_source.with(headers: headers.normalize)
						SourceWithDeadline.new(source, @deadline)
					)
				end
			end


@@ 227,8 270,9 @@ module Dhall
					resolver:    self,
					relative_to: Dhall::Import::RelativePath.new
				).then do |headers|
					source = https_source.with(headers: headers.normalize)
					@https_resolutions.register(
						https_source.with(headers: headers.normalize)
						SourceWithDeadline.new(source, @deadline)
					)
				end
			end


@@ 341,7 385,7 @@ module Dhall
			def resolve_raw(resolver:, relative_to:)
				real_path = @expr.real_path(relative_to)
				real_path.resolve(resolver).then do |result|
					@expr.parse_and_check(result).resolve(
					@expr.parse_and_check(result, deadline: resolver.deadline).resolve(
						resolver:    resolver.child(real_path),
						relative_to: real_path
					)

M lib/dhall/util.rb => lib/dhall/util.rb +42 -0
@@ 1,5 1,7 @@
# frozen_string_literal: true

require "timeout"

module Dhall
	module Util
		class AllOf


@@ 61,6 63,46 @@ module Dhall
			end
		end

		class Deadline
			def self.for_timeout(timeout)
				if timeout.nil? || timeout.to_f.infinite?
					NoDeadline.new
				else
					new(Time.now + timeout)
				end
			end

			def initialize(deadline)
				@deadline = deadline
			end

			def exceeded?
				@deadline < Time.now
			end

			def timeout
				[0.000000000000001, @deadline - Time.now].max
			end

			def timeout_block(&block)
				Timeout.timeout(timeout, TimeoutException, &block)
			end
		end

		class NoDeadline
			def exceeded?
				false
			end

			def timeout
				nil
			end

			def timeout_block
				yield
			end
		end

		def self.match_results(xs=nil, ys=nil)
			Array(xs).each_with_index.map do |r, idx|
				yield r, ys[idx]

M test/test_load.rb => test/test_load.rb +44 -6
@@ 1,5 1,6 @@
# frozen_string_literal: true

require "securerandom"
require "minitest/autorun"

require "dhall"


@@ 19,12 20,6 @@ class TestLoad < Minitest::Test
		end
	end

	def test_load_invalid_utf8_binary_input
		assert_raises ArgumentError do
			Dhall.load("\xc3\x28".b).sync
		end
	end

	def test_load_natural_binary
		assert_equal(
			Dhall::Natural.new(value: 1),


@@ 66,4 61,47 @@ class TestLoad < Minitest::Test
			Dhall.load_raw("1 + \"hai\"")
		)
	end

	def test_load_parse_timeout
		Dhall::Parser.stub(:parse, ->(*) { sleep 1 }) do
			assert_raises Dhall::TimeoutException do
				Dhall.load("./start", timeout: 0.1).sync
			end
		end
	end

	def test_load_normalize_timeout
		ack = <<~DHALL
			let iter =
				λ(f : Natural → Natural) → λ(n : Natural) →
				Natural/fold n Natural f (f (1))
			in
				λ(m : Natural) →
					Natural/fold m (Natural → Natural) iter (λ(x : Natural) → x + 1)
		DHALL
		assert_raises Dhall::TimeoutException do
			Dhall.load("(#{ack}) 10 10", timeout: 0.1).sync
		end
	end

	def test_load_resolve_timeout
		resolver = Dhall::Resolvers::LocalOnly.new(
			path_reader: lambda do |sources|
				sources.map do |_|
					Promise.resolve(nil).then do
						sleep 0.1
						"./#{SecureRandom.hex}"
					end
				end
			end
		)

		assert_raises Dhall::TimeoutException do
			Dhall.load(
				"./start",
				resolver: resolver,
				timeout:  0.1
			).sync
		end
	end
end

M test/test_resolvers.rb => test/test_resolvers.rb +10 -10
@@ 9,61 9,61 @@ class TestResolvers < Minitest::Test
	def test_default_resolver_path
		resolver = Dhall::Resolvers::Default.new(
			path_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
				sources.map { |source| Promise.resolve(source.to_s) }
			end
		)
		source = Dhall::Import::AbsolutePath.new("dhall", "common", "x.dhall")
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
		assert_equal source.to_s, promise.sync
	end

	def test_default_resolver_env
		resolver = Dhall::Resolvers::Default.new(
			environment_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
				sources.map { |source| Promise.resolve(source.to_s) }
			end
		)
		source = Dhall::Import::EnvironmentVariable.new("__DHALL_IMPORT_TEST")
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
		assert_equal source.to_s, promise.sync
	end

	def test_default_resolver_http
		resolver = Dhall::Resolvers::Default.new(
			http_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
				sources.map { |source| Promise.resolve(source.to_s) }
			end
		)
		source = Dhall::Import::Http.new(nil, "example.com", "x.dhall", nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
		assert_equal source.to_s, promise.sync
	end

	def test_default_resolver_https
		resolver = Dhall::Resolvers::Default.new(
			https_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
				sources.map { |source| Promise.resolve(source.to_s) }
			end
		)
		source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
		assert_equal source.to_s, promise.sync
	end

	def test_default_resolver_https_uses_http
		resolver = Dhall::Resolvers::Default.new(
			http_reader: lambda do |sources|
				sources.map { |source| Promise.resolve(source) }
				sources.map { |source| Promise.resolve(source.to_s) }
			end
		)
		source = Dhall::Import::Https.new(nil, "example.com", "x.dhall", nil)
		promise = source.resolve(resolver)
		resolver.finish!
		assert_equal source, promise.sync
		assert_equal source.to_s, promise.sync
	end

	def test_local_only_resolver_rejects_http