sharpetronics/httpx
1
0
Fork 0
mirror of https://github.com/HoneyryderChuck/httpx.git synced 2025-12-15 00:01:02 -05:00
Code Issues Projects Releases Wiki Activity

Merge branch 'issue-186' into 'master'

Browse Source 
native resolver:  resolv.conf search option

Closes #186

See merge request honeyryderchuck/httpx!198
This commit is contained in:
HoneyryderChuck 2022-03-29 22:06:25 +00:00
commit 2a9f56cb44
21 changed files with 173 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.gitlab-ci.yml
View File

@ -158,3 +158,33 @@ pages:
only:
- master
- blog
prepare_release:
stage: prepare
rules:
- if: $CI_COMMIT_TAG
when: never
script:
- echo "EXTRA_DESCRIPTION=$(cat doc/release_notes/${${CI_COMMIT_TAG:1}//./_}.md)" >> variables.env
- echo "TAG=v$(cat CI_COMMIT_TAG)" >> variables.env
artifacts:
reports:
dotenv: variables.env
release:
stage: deploy
image: registry.gitlab.com/gitlab-org/release-cli:latest
needs:
- job: prepare_release
artifacts: true
rules:
- if: $CI_COMMIT_TAG
when: never
script:
- echo "running release_job for $TAG"
release:
name: 'Release $TAG'
description: '$EXTRA_DESCRIPTION'
tag_name: '$TAG'
ref: '$CI_COMMIT_SHA'

5
doc/release_notes/0_19_4.md
View File

@ -1,4 +1,4 @@
# 0.19.3
# 0.19.4
## Improvements
@ -10,4 +10,5 @@ The (optional) FFI-based TLS module for jruby was deleted. Besides it being cumb
* `webmock` integration was fixed to take the mocked URI query string into account.
* fix internal codepath where mergeable-but-not-coalescable connections were still triggering the coalesce branch.
* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
* fixed after-use mutation of connection addresses array which was making it empty after initial usage.
* fixed a "busy loop" caused by long-running native resolver not signaling it had "nothing to do".

2
lib/httpx/headers.rb
View File

@ -42,6 +42,8 @@ module HTTPX
def same_headers?(headers)
@headers.empty? || begin
headers.each do |k, v|
next unless key?(k)
return false unless v == self[k]
end
true

5
lib/httpx/options.rb
View File

@ -209,8 +209,9 @@ module HTTPX
ivars.all? do |ivar|
case ivar
when :@headers
headers = instance_variable_get(ivar)
headers.same_headers?(other.instance_variable_get(ivar))
# currently, this is used to pick up an available matching connection.
# the headers do not play a role, as they are relevant only for the request.
true
when *REQUEST_IVARS
true
else

6
lib/httpx/plugins/proxy.rb
View File

@ -174,12 +174,6 @@ module HTTPX
@origin.port = proxy_uri.port
end
def match?(uri, options)
return super unless @options.proxy
super && @options.proxy == options.proxy
end
def coalescable?(connection)
return super unless @options.proxy

6
lib/httpx/plugins/proxy/ssh.rb
View File

@ -78,12 +78,6 @@ module HTTPX
end
module ConnectionMethods
def match?(uri, options)
return super unless @options.proxy
super && @options.proxy == options.proxy
end
# should not coalesce connections here, as the IP is the IP of the proxy
def coalescable?(*)
return super unless @options.proxy

46
lib/httpx/resolver/https.rb
View File

@ -11,6 +11,15 @@ module HTTPX
using URIExtensions
using StringExtensions
module DNSExtensions
refine Resolv::DNS do
def generate_candidates(name)
@config.generate_candidates(name)
end
end
end
using DNSExtensions
NAMESERVER = "https://1.1.1.1/dns-query"
DEFAULTS = {
@ -76,30 +85,37 @@ module HTTPX
if hostname.nil?
hostname = connection.origin.host
log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
hostname = @resolver.generate_candidates(hostname).each do |name|
@queries[name.to_s] = connection
end.first.to_s
else
@queries[hostname] = connection
end
log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
begin
request = build_request(hostname)
request.on(:response, &method(:on_response).curry(2)[request])
request.on(:promise, &method(:on_promise))
@requests[request] = connection
@requests[request] = hostname
resolver_connection.send(request)
@queries[hostname] = connection
@connections << connection
rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
emit_resolve_error(connection, hostname, e)
@queries.delete(hostname)
emit_resolve_error(connection, connection.origin.host, e)
end
end
def on_response(request, response)
response.raise_for_status
rescue StandardError => e
connection = @requests[request]
hostname = @queries.key(connection)
emit_resolve_error(connection, hostname, e)
hostname = @requests.delete(request)
connection = @queries.delete(hostname)
emit_resolve_error(connection, connection.origin.host, e)
else
# @type var response: HTTPX::Response
parse(response)
parse(request, response)
ensure
@requests.delete(request)
end
@ -109,20 +125,21 @@ module HTTPX
stream.refuse
end
def parse(response)
def parse(request, response)
begin
answers = decode_response_body(response)
rescue Resolv::DNS::DecodeError, JSON::JSONError => e
host, connection = @queries.first
@queries.delete(host)
emit_resolve_error(connection, host, e)
emit_resolve_error(connection, connection.origin.host, e)
return
end
if answers.nil? || answers.empty?
host, connection = @queries.first
@queries.delete(host)
emit_resolve_error(connection, host)
host = @requests.delete(request)
connection = @queries.delete(host)
emit_resolve_error(connection)
return
else
answers = answers.group_by { |answer| answer["name"] }
answers.each do |hostname, addresses|
@ -130,7 +147,6 @@ module HTTPX
if address.key?("alias")
alias_address = answers[address["alias"]]
if alias_address.nil?
connection = @queries[hostname]
@queries.delete(address["name"])
if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
@connections.delete(connection)
@ -152,6 +168,10 @@ module HTTPX
next unless connection # probably a retried query for which there's an answer
@connections.delete(connection)
# eliminate other candidates
@queries.delete_if { |_, conn| connection == conn }
Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
end

81
lib/httpx/resolver/native.rb
View File

@ -46,6 +46,8 @@ module HTTPX
@ns_index = 0
@resolver_options = DEFAULTS.merge(@options.resolver_options)
@nameserver = @resolver_options[:nameserver]
@ndots = @resolver_options[:ndots]
@search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
@_timeouts = Array(@resolver_options[:timeouts])
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@connections = []
@ -136,33 +138,32 @@ module HTTPX
return if @queries.empty? || !@start_timeout
loop_time = Utils.elapsed_time(@start_timeout)
connections = []
queries = {}
while (query = @queries.shift)
h, connection = query
host = connection.origin.host
timeout = (@timeouts[host][0] -= loop_time)
unless timeout.negative?
queries[h] = connection
next
end
@timeouts[host].shift
if @timeouts[host].empty?
@timeouts.delete(host)
@connections.delete(connection)
# This loop_time passed to the exception is bogus. Ideally we would pass the total
# resolve timeout, including from the previous retries.
raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
# raise NativeResolveError.new(connection, host)
else
log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
connections << connection
queries[h] = connection
end
query = @queries.first
return unless query
h, connection = query
host = connection.origin.host
timeout = (@timeouts[host][0] -= loop_time)
return unless timeout.negative?
@timeouts[host].shift
if @timeouts[host].empty?
@timeouts.delete(host)
@queries.delete(h)
return unless @queries.empty?
@connections.delete(connection)
# This loop_time passed to the exception is bogus. Ideally we would pass the total
# resolve timeout, including from the previous retries.
raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
else
log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
resolve(connection)
end
@queries = queries
connections.each { |ch| resolve(ch) }
end
def dread(wsize = @resolver_options[:packet_size])
@ -194,7 +195,7 @@ module HTTPX
@queries.delete(hostname)
@timeouts.delete(hostname)
@connections.delete(connection)
ex = NativeResolveError.new(connection, hostname, e.message)
ex = NativeResolveError.new(connection, connection.origin.host, e.message)
ex.set_backtrace(e.backtrace)
raise ex
end
@ -203,9 +204,11 @@ module HTTPX
hostname, connection = @queries.first
@queries.delete(hostname)
@timeouts.delete(hostname)
@connections.delete(connection)
raise NativeResolveError.new(connection, hostname)
unless @queries.value?(connection)
@connections.delete(connection)
raise NativeResolveError.new(connection, connection.origin.host)
end
else
address = addresses.first
name = address["name"]
@ -224,6 +227,9 @@ module HTTPX
connection = @queries.delete(name)
end
# eliminate other candidates
@queries.delete_if { |_, conn| connection == conn }
if address.key?("alias") # CNAME
# clean up intermediate queries
@timeouts.delete(name) unless connection.origin.host == name
@ -256,8 +262,13 @@ module HTTPX
if hostname.nil?
hostname = connection.origin.host
log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
hostname = generate_candidates(hostname).each do |name|
@queries[name] = connection
end.first
else
@queries[hostname] = connection
end
@queries[hostname] = connection
log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
begin
@write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
@ -266,6 +277,18 @@ module HTTPX
end
end
def generate_candidates(name)
return [name] if name.end_with?(".")
candidates = []
name_parts = name.scan(/[^.]+/)
candidates = [name] if @ndots <= name_parts.size - 1
candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
candidates << name unless candidates.include?(name)
candidates
end
def build_socket
return if @io

2
lib/httpx/resolver/system.rb
View File

@ -82,6 +82,8 @@ module HTTPX
_, connection = @queries.first
return unless connection
@timeouts[connection.origin.host].first
end

4
sig/resolver/https.rbs
View File

@ -8,7 +8,7 @@ module HTTPX
@family: ip_family
@options: Options
@requests: Hash[Request, Connection]
@requests: Hash[Request, String]
@connections: Array[Connection]
@uri: URI::Generic
@uri_addresses: Array[String]?
@ -29,7 +29,7 @@ module HTTPX
def on_response: (Request, response) -> void
def parse: (Response response) -> void
def parse: (Request request, Response response) -> void
def build_request: (String hostname) -> Request

2
sig/resolver/native.rbs
View File

@ -46,6 +46,8 @@ module HTTPX
def resolve: (?Connection connection, ?String hostname) -> void
def generate_candidates: (String) -> Array[String]
def build_socket: () -> void
def transition: (Symbol nextstate) -> void

14
test/http_test.rb
View File

@ -72,9 +72,7 @@ class HTTPTest < Minitest::Test
end
def test_max_streams
server = KeepAliveServer.new
th = Thread.new { server.start }
begin
start_test_servlet(KeepAliveServer) do |server|
uri = "#{server.origin}/2"
HTTPX.plugin(SessionWithPool).with(max_concurrent_requests: 1).wrap do |http|
responses = http.get(uri, uri, uri)
@ -82,16 +80,11 @@ class HTTPTest < Minitest::Test
connection_count = http.pool.connection_count
assert connection_count == 2, "expected to have 2 connections, instead have #{connection_count}"
end
ensure
server.shutdown
th.join
end
end
def test_trailers
server = HTTPTrailersServer.new
th = Thread.new { server.start }
begin
start_test_servlet(HTTPTrailersServer) do |server|
uri = "#{server.origin}/"
HTTPX.plugin(SessionWithPool).wrap do |http|
response = http.get(uri)
@ -100,9 +93,6 @@ class HTTPTest < Minitest::Test
verify_header(response.headers, "x-trailer", "hello")
verify_header(response.headers, "x-trailer-2", "world")
end
ensure
server.shutdown
th.join
end
end

18
test/https_test.rb
View File

@ -136,25 +136,20 @@ class HTTPSTest < Minitest::Test
end
def test_http2_client_sends_settings_timeout
server = SettingsTimeoutServer.new
th = Thread.new { server.accept }
begin
test_server = nil
start_test_servlet(SettingsTimeoutServer) do |server|
test_server = server
uri = "#{server.origin}/"
http = HTTPX.plugin(SessionWithPool).with(timeout: { settings_timeout: 3 }, ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
response = http.get(uri)
verify_error_response(response, HTTPX::SettingsTimeoutError)
ensure
server.shutdown
th.join
end
last_frame = server.frames.last
last_frame = test_server.frames.last
assert last_frame[:error] == :settings_timeout
end
def test_http2_client_goaway_with_no_response
server = KeepAlivePongServer.new
th = Thread.new { server.accept }
begin
start_test_servlet(KeepAlivePongServer) do |server|
uri = "#{server.origin}/"
HTTPX.plugin(SessionWithPool).with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE }) do |http|
response = http.get(uri)
@ -162,9 +157,6 @@ class HTTPSTest < Minitest::Test
response = http.get(uri)
verify_error_response(response, HTTPX::Connection::HTTP2::GoawayError)
end
ensure
server.shutdown
th.join
end
end
end

9
test/response_test.rb
View File

@ -7,8 +7,13 @@ class ResponseTest < Minitest::Test
include ResponseHelpers
if RUBY_VERSION >= "3.0.0"
require_relative "extensions/response_pattern_match"
include ResponsePatternMatchTests
begin
eval("case 1; in 1 ;then true; end") # rubocop:disable Style/EvalWithLocation
require_relative "extensions/response_pattern_match"
include ResponsePatternMatchTests
rescue SyntaxError
# truffleruby advertises ruby 3 support, but still hasn't implemented pattern matching
end
end
def test_response_status

18
test/support/assertion_helpers.rb
View File

@ -38,7 +38,7 @@ module ResponseHelpers
delta += if RUBY_ENGINE == "truffleruby"
# truffleruby has a hard time complying reliably with this delta when running in parallel. Therefore,
# we give it a bit of leeway.
10
20
else
# delta checks become very innacurate under multi-thread mode, and elapsed time. we give it some leeway too.
3
@ -98,4 +98,20 @@ module ResponseHelpers
def fixture_file_path
File.join("test", "support", "fixtures", fixture_file_name)
end
def start_test_servlet(servlet_class)
server = servlet_class.new
th = Thread.new { server.start }
begin
yield server
ensure
server.shutdown
begin
Timeout.timeout(3) { th.join }
rescue Timeout::Error
th.kill
end
end
end
end

7
test/support/requests/plugins/authentication.rb
View File

@ -49,9 +49,7 @@ module Requests
def test_plugin_ntlm_authentication
return if origin.start_with?("https")
server = NTLMServer.new
th = Thread.new { server.start }
begin
start_test_servlet(NTLMServer) do |server|
uri = "#{server.origin}/"
HTTPX.plugin(SessionWithPool).plugin(:ntlm_authentication).wrap do |http|
# skip unless NTLM
@ -65,9 +63,6 @@ module Requests
# invalid_response = http.ntlm_authentication("user", "fake").get(uri)
# verify_status(invalid_response, 401)
end
ensure
server.shutdown
th.join
end
end

7
test/support/requests/plugins/compression.rb
View File

@ -90,18 +90,13 @@ module Requests
# run this only for http/1.1 mode, as this is a local test server
return unless origin.start_with?("http://")
server = NoContentLengthServer.new
th = Thread.new { server.start }
begin
start_test_servlet(NoContentLengthServer) do |server|
http = HTTPX.plugin(:compression)
uri = build_uri("/", server.origin)
response = http.get(uri)
verify_status(response, 200)
body = response.body.to_s
assert body == "helloworld"
ensure
server.shutdown
th.join
end
end

14
test/support/requests/plugins/expect.rb
View File

@ -17,9 +17,7 @@ module Requests
# run this only for http/1.1 mode, as this is a local test server
return unless origin.start_with?("http://")
server = Expect100Server.new
th = Thread.new { server.start }
begin
start_test_servlet(Expect100Server) do |server|
http = HTTPX.plugin(:expect)
uri = build_uri("/delay?delay=4", server.origin)
response = http.post(uri, body: "helloworld")
@ -30,9 +28,6 @@ module Requests
next_request = http.build_request(:post, build_uri("/", server.origin), body: "helloworld")
verify_header(next_request.headers, "expect", "100-continue")
ensure
server.shutdown
th.join
end
end
@ -54,9 +49,7 @@ module Requests
# run this only for http/1.1 mode, as this is a local test server
return unless origin.start_with?("http://")
server = Expect100Server.new
th = Thread.new { server.start }
begin
start_test_servlet(Expect100Server) do |server|
http = HTTPX.plugin(:expect)
uri = build_uri("/no-expect", server.origin)
response = http.post(uri, body: "helloworld")
@ -67,9 +60,6 @@ module Requests
next_request = http.build_request(:post, build_uri("/", server.origin), body: "helloworld")
verify_no_header(next_request.headers, "expect")
ensure
server.shutdown
th.join
end
end

15
test/support/requests/plugins/persistent.rb
View File

@ -40,23 +40,18 @@ module Requests
def test_persistent_retry_http2_goaway
return unless origin.start_with?("https")
server = KeepAlivePongServer.new
th = Thread.new { server.accept }
http = HTTPX.plugin(SessionWithPool)
.plugin(RequestInspector)
.plugin(:persistent) # implicit max_retries == 1
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
begin
start_test_servlet(KeepAlivePongServer) do |server|
http = HTTPX.plugin(SessionWithPool)
.plugin(RequestInspector)
.plugin(:persistent) # implicit max_retries == 1
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
uri = "#{server.origin}/"
response = http.get(uri)
verify_status(response, 200)
response = http.get(uri)
verify_status(response, 200)
assert http.calls == 2, "expect request to be built 2 times (was #{http.calls})"
ensure
http.close
server.shutdown
th.join
end
end
end

4
test/support/requests/resolvers.rb
View File

@ -81,7 +81,9 @@ module Requests
# this test mocks an unresponsive DNS server which doesn't return a DNS asnwer back.
define_method :"test_resolver_#{resolver}_timeout" do
session = HTTPX.plugin(SessionWithPool)
uri = build_uri("/get")
uri = URI(build_uri("/get"))
# absolute URL, just to shorten the impact of resolv.conf search.
uri.host = "#{uri.host}."
resolver_class = Class.new(HTTPX::Resolver::Native) do
def interests

2
test/support/servlets/test.rb
View File

@ -54,7 +54,7 @@ class TestHTTP2Server
@server.close
end
def accept
def start
begin
loop do
sock = @server.accept