Compare commits

..

No commits in common. "master" and "v1.6.0" have entirely different histories.

20 changed files with 60 additions and 295 deletions

View File

@ -1,17 +0,0 @@
# 1.6.1
## Improvements
* `:oauth` plugin: `.oauth_session` can be called with an `:audience` parameter, which has the effect of adding it as an extra form body parameter of the token request.
## Bugfixes
* options: when freezing the options, skip freezing `:debug`; it's usually a file/IO/stream object (stdout, stderr...), which makes it error when log messages are written.
* tcp: fixed adding IPv6 addresses to a tcp object when IPv4 connection probe is ongoing so that the next try uses the first ipv6 address.
* tcp: reorder addresses on reconnection, so ipv6 is tried first in case it is still valid.
* tcp: make sure ip index is decremented on error, so the next tried IP may be a valid one.
* tcp: do not reattempt connecting if there are no available addresses to connect. This may happen in a fiber-aware context, where fiber A waits on connection, fiber B reconnects as a result on an error or GOAWAY frame and waits on the resolver DNS answer, and when context is passed back to fiber B, it should go back to the invalidate the response and try again while waiting on the resolver as well.
* ssl: on connection coalescing, do not merge the ssl sessions, as these are frozen post-initialization.
* http2: all received GOAWAY frames emit goaway error and teardown the connection independent of the error code (it was only doing it for `:noerror`, but others may appear).
* do not check at require time whether the network is multi-homed; instead, defer it to first use and cache (this can break environments which block access to certain syscalls during boot time).
* options: do not ignore when user sets `:ip_families` in name resolution.

View File

@ -44,7 +44,7 @@ module HTTPX
attr_accessor :current_session, :family
protected :ssl_session, :sibling
protected :sibling
def initialize(uri, options)
@current_session = @current_selector =
@ -177,7 +177,7 @@ module HTTPX
def merge(connection)
@origins |= connection.instance_variable_get(:@origins)
if @ssl_session.nil? && connection.ssl_session
if connection.ssl_session
@ssl_session = connection.ssl_session
@io.session_new_cb do |sess|
@ssl_session = sess

View File

@ -23,8 +23,8 @@ module HTTPX
end
class GoawayError < Error
def initialize(code = :no_error)
super(0, code)
def initialize
super(0, :no_error)
end
end
@ -385,10 +385,12 @@ module HTTPX
while (request = @pending.shift)
emit(:error, request, error)
end
else
ex = GoawayError.new(error)
when :no_error
ex = GoawayError.new
@pending.unshift(*@streams.keys)
teardown
else
ex = Error.new(0, error)
end
if ex

View File

@ -14,10 +14,7 @@ module HTTPX
def initialize(origin, addresses, options)
@state = :idle
@keep_open = false
@addresses = []
@ip_index = -1
@ip = nil
@hostname = origin.host
@options = options
@fallback_protocol = @options.fallback_protocol
@ -56,23 +53,19 @@ module HTTPX
@addresses = [*@addresses[0, ip_index], *addrs, *@addresses[ip_index..-1]]
else
@addresses.unshift(*addrs)
@ip_index += addrs.size if @ip_index
end
@ip_index += addrs.size
end
# eliminates expired entries and returns whether there are still any left.
def addresses?
prev_addr_size = @addresses.size
@addresses.delete_if(&:expired?).sort! do |addr1, addr2|
if addr1.ipv6?
addr2.ipv6? ? 0 : 1
else
addr2.ipv6? ? -1 : 0
end
end
@addresses.delete_if(&:expired?)
@ip_index = @addresses.size - 1 if prev_addr_size != @addresses.size
unless (decr = prev_addr_size - @addresses.size).zero?
@ip_index = @addresses.size - decr
end
@addresses.any?
end
@ -88,17 +81,6 @@ module HTTPX
def connect
return unless closed?
if @addresses.empty?
# an idle connection trying to connect with no available addresses is a connection
# out of the initial context which is back to the DNS resolution loop. This may
# happen in a fiber-aware context where a connection reconnects with expired addresses,
# and context is passed back to a fiber on the same connection while waiting for the
# DNS answer.
log { "tried connecting while resolving, skipping..." }
return
end
if !@io || @io.closed?
transition(:idle)
@io = build_socket
@ -106,33 +88,29 @@ module HTTPX
try_connect
rescue Errno::EHOSTUNREACH,
Errno::ENETUNREACH => e
@ip_index -= 1
raise e if @ip_index.negative?
raise e if @ip_index <= 0
log { "failed connecting to #{@ip} (#{e.message}), evict from cache and trying next..." }
Resolver.cached_lookup_evict(@hostname, @ip)
@ip_index -= 1
@io = build_socket
retry
rescue Errno::ECONNREFUSED,
Errno::EADDRNOTAVAIL,
SocketError,
IOError => e
@ip_index -= 1
raise e if @ip_index.negative?
raise e if @ip_index <= 0
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
@ip_index -= 1
@io = build_socket
retry
rescue Errno::ETIMEDOUT => e
@ip_index -= 1
raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index.negative?
raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
log { "failed connecting to #{@ip} (#{e.message}), trying next..." }
@ip_index -= 1
@io = build_socket
retry
end

View File

@ -1,5 +1,7 @@
# frozen_string_literal: true
require "socket"
module HTTPX
# Contains a set of options which are passed and shared across from session to its requests or
# responses.
@ -149,14 +151,6 @@ module HTTPX
def freeze
self.class.options_names.each do |ivar|
# avoid freezing debug option, as when it's set, it's usually an
# object which cannot be frozen, like stderr or stdout. It's a
# documented exception then, and still does not defeat the purpose
# here, which is to make option objects shareable across ractors,
# and in most cases debug should be nil, or one of the objects
# which will eventually be shareable, like STDOUT or STDERR.
next if ivar == :debug
instance_variable_get(:"@#{ivar}").freeze
end
super
@ -412,6 +406,18 @@ module HTTPX
end
end
# https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
ip_address_families = begin
list = Socket.ip_address_list
if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
[Socket::AF_INET6, Socket::AF_INET]
else
[Socket::AF_INET]
end
rescue NotImplementedError
[Socket::AF_INET]
end.freeze
DEFAULT_OPTIONS = {
:max_requests => Float::INFINITY,
:debug => nil,
@ -456,7 +462,7 @@ module HTTPX
:resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
:resolver_options => { cache: true }.freeze,
:pool_options => EMPTY_HASH,
:ip_families => nil,
:ip_families => ip_address_families,
:close_on_fork => false,
}.freeze
end

View File

@ -16,7 +16,7 @@ module HTTPX
SUPPORTED_AUTH_METHODS = %w[client_secret_basic client_secret_post].freeze
class OAuthSession
attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope, :audience
attr_reader :grant_type, :client_id, :client_secret, :access_token, :refresh_token, :scope
def initialize(
issuer:,
@ -25,7 +25,6 @@ module HTTPX
access_token: nil,
refresh_token: nil,
scope: nil,
audience: nil,
token_endpoint: nil,
response_type: nil,
grant_type: nil,
@ -42,7 +41,6 @@ module HTTPX
when Array
scope
end
@audience = audience
@access_token = access_token
@refresh_token = refresh_token
@token_endpoint_auth_method = String(token_endpoint_auth_method) if token_endpoint_auth_method
@ -127,11 +125,7 @@ module HTTPX
grant_type = oauth_session.grant_type
headers = {}
form_post = {
"grant_type" => grant_type,
"scope" => Array(oauth_session.scope).join(" "),
"audience" => oauth_session.audience,
}.compact
form_post = { "grant_type" => grant_type, "scope" => Array(oauth_session.scope).join(" ") }.compact
# auth
case oauth_session.token_endpoint_auth_method

View File

@ -1,6 +1,5 @@
# frozen_string_literal: true
require "socket"
require "resolv"
module HTTPX
@ -23,20 +22,6 @@ module HTTPX
module_function
def supported_ip_families
@supported_ip_families ||= begin
# https://github.com/ruby/resolv/blob/095f1c003f6073730500f02acbdbc55f83d70987/lib/resolv.rb#L408
list = Socket.ip_address_list
if list.any? { |a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? }
[Socket::AF_INET6, Socket::AF_INET]
else
[Socket::AF_INET]
end
rescue NotImplementedError
[Socket::AF_INET]
end.freeze
end
def resolver_for(resolver_type, options)
case resolver_type
when Symbol

View File

@ -15,9 +15,7 @@ module HTTPX
@options = options
@resolver_options = @options.resolver_options
ip_families = options.ip_families || Resolver.supported_ip_families
@resolvers = ip_families.map do |ip_family|
@resolvers = options.ip_families.map do |ip_family|
resolver = resolver_type.new(ip_family, options)
resolver.multi = self
resolver
@ -69,12 +67,8 @@ module HTTPX
addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
return false unless addresses
ip_families = connection.options.ip_families || Resolver.supported_ip_families
resolved = false
addresses.group_by(&:family).sort { |(f1, _), (f2, _)| f2 <=> f1 }.each do |family, addrs|
next unless ip_families.include?(family)
# try to match the resolver by family. However, there are cases where that's not possible, as when
# the system does not have IPv6 connectivity, but it does support IPv6 via loopback/link-local.
resolver = @resolvers.find { |r| r.family == family } || @resolvers.first
@ -91,11 +85,7 @@ module HTTPX
end
def lazy_resolve(connection)
ip_families = connection.options.ip_families || Resolver.supported_ip_families
@resolvers.each do |resolver|
next unless ip_families.include?(resolver.family)
resolver << @current_session.try_clone_connection(connection, @current_selector, resolver.family)
next if resolver.empty?

View File

@ -79,18 +79,12 @@ module HTTPX
"answer #{connection.peer.host}: #{addresses.inspect} (early resolve: #{early_resolve})"
end
# do not apply resolution delay for non-dns name resolution
if !early_resolve &&
# just in case...
@current_selector &&
# resolution delay only applies to IPv4
family == Socket::AF_INET &&
# connection already has addresses and initiated/ended handshake
!connection.io &&
# no need to delay if not supporting dual stack / multi-homed IP
(connection.options.ip_families || Resolver.supported_ip_families).size > 1 &&
# connection URL host is already the IP (early resolve included perhaps?)
addresses.first.to_s != connection.peer.host.to_s
if !early_resolve && # do not apply resolution delay for non-dns name resolution
@current_selector && # just in case...
family == Socket::AF_INET && # resolution delay only applies to IPv4
!connection.io && # connection already has addresses and initiated/ended handshake
connection.options.ip_families.size > 1 && # no need to delay if not supporting dual stack IP
addresses.first.to_s != connection.peer.host.to_s # connection URL host is already the IP (early resolve included perhaps?)
log { "resolver #{FAMILY_TYPES[RECORD_TYPES[family]]}: applying resolution delay..." }
@current_selector.after(0.05) do

View File

@ -187,9 +187,7 @@ module HTTPX
transition(:open)
ip_families = connection.options.ip_families || Resolver.supported_ip_families
ip_families.each do |family|
connection.options.ip_families.each do |family|
@queries << [family, connection]
end
async_resolve(connection, hostname, scheme)
@ -197,7 +195,7 @@ module HTTPX
end
def async_resolve(connection, hostname, scheme)
families = connection.options.ip_families || Resolver.supported_ip_families
families = connection.options.ip_families
log { "resolver: query for #{hostname}" }
timeouts = @timeouts[connection.peer.host]
resolve_timeout = timeouts.first

View File

@ -1,5 +1,5 @@
# frozen_string_literal: true
module HTTPX
VERSION = "1.6.1"
VERSION = "1.6.0"
end

View File

@ -1,42 +0,0 @@
# frozen_string_literal: true
require "test_helper"
require "support/http_helpers"
require "webmock/minitest"
class Bug_1_6_1_Test < Minitest::Test
include HTTPHelpers
def test_retries_should_retry_on_goaway_cancel
start_test_servlet(GoawayCancelErrorServer) do |server|
http = HTTPX.plugin(SessionWithPool)
.plugin(RequestInspector)
.plugin(:retries)
.with(ssl: { verify_mode: OpenSSL::SSL::VERIFY_NONE })
uri = "#{server.origin}/"
response = http.get(uri)
verify_status(response, 200)
assert http.calls == 1, "expect request to be built 1 more time (was #{http.calls})"
http.close
end
end
class GoawayCancelErrorServer < TestHTTP2Server
def initialize(**)
@sent = Hash.new(false)
super
end
private
def handle_stream(conn, stream)
if @cancelled
super
else
conn.goaway(:cancel)
@cancelled = true
end
end
end
end

View File

@ -106,7 +106,7 @@ module HTTPX
end
class GoawayError < Error
def initialize: (?Symbol code) -> void
def initialize: () -> void
end
class PingError < Error

View File

@ -130,7 +130,7 @@ module HTTPX
attr_reader pool_options: pool_options
# ip_families
attr_reader ip_families: Array[ip_family]?
attr_reader ip_families: Array[ip_family]
def ==: (Options other) -> bool
@ -195,7 +195,7 @@ module HTTPX
def option_addresses: (ipaddr | _ToAry[ipaddr] value) -> Array[ipaddr]
def option_ip_families: (ip_family | _ToAry[ip_family] value) -> Array[ip_family]
def option_ip_families: (Integer | _ToAry[Integer] value) -> Array[Integer]
end
type options = Options | Hash[Symbol, untyped]

View File

@ -27,21 +27,7 @@ module HTTPX
attr_reader scope: Array[String]?
attr_reader audience: String?
def initialize: (
issuer: uri,
client_id: String,
client_secret: String,
?access_token: String?,
?refresh_token: String?,
?scope: (Array[String] | String)?,
?token_endpoint: String?,
?response_type: String?,
?grant_type: String?,
?token_endpoint_auth_method: ::String,
?audience: ::String
) -> void
def initialize: (issuer: uri, client_id: String, client_secret: String, ?access_token: String?, ?refresh_token: String?, ?scope: (Array[String] | String)?, ?token_endpoint: String?, ?response_type: String?, ?grant_type: String?, ?token_endpoint_auth_method: ::String) -> void
def token_endpoint: () -> String

View File

@ -23,8 +23,6 @@ module HTTPX
def self?.hosts_resolve: (String hostname) -> Array[Entry]?
def self?.supported_ip_families: () -> Array[ip_family]
def self?.resolver_for: (Symbol | singleton(Resolver) resolver_type, Options options) -> singleton(Resolver)
def self?.cached_lookup: (String hostname) -> Array[Entry]?

View File

@ -1,63 +0,0 @@
# frozen_string_literal: true
require "tempfile"
require_relative "../test_helper"
class TCPTest < Minitest::Test
include HTTPX
def test_tcp_ip_index_rebalance_on_new_addresses
origin = URI("http://example.com")
options = Options.new
tcp_class = Class.new(TCP) do
attr_accessor :ip_index
end
# initialize with no addresses, ip index points nowhere
tcp = tcp_class.new(origin, [], options)
assert tcp.ip_index == -1
# initialize with addresses, ip index points to the last element
tcp1 = tcp_class.new(origin, [Resolver::Entry.new("127.0.0.1")], options)
assert tcp1.addresses == ["127.0.0.1"]
assert tcp1.ip_index.zero?
tcp2 = tcp_class.new(origin, [Resolver::Entry.new("127.0.0.1"), Resolver::Entry.new("127.0.0.2")], options)
assert tcp2.addresses == ["127.0.0.1", "127.0.0.2"]
assert tcp2.ip_index == 1
tcp3 = tcp_class.new(origin, [Resolver::Entry.new("::1")], options)
assert tcp3.addresses == ["::1"]
assert tcp3.ip_index.zero?
# add addresses, ip index must point to previous ip after address expansion
tcp.add_addresses([Resolver::Entry.new("::1")])
assert tcp.addresses == ["::1"]
assert tcp.ip_index.zero?
tcp1.add_addresses([Resolver::Entry.new("::1")])
assert tcp1.addresses == ["::1", "127.0.0.1"]
assert tcp1.ip_index == 1
# makes the ipv6 address the next address to try
tcp2.add_addresses([Resolver::Entry.new("::1")])
assert tcp2.addresses == ["127.0.0.1", "::1", "127.0.0.2"]
assert tcp2.ip_index == 2
tcp3.add_addresses([Resolver::Entry.new("127.0.0.1")])
assert tcp3.addresses == ["127.0.0.1", "::1"]
assert tcp3.ip_index == 1
tcp3.add_addresses([Resolver::Entry.new("::2")])
assert tcp3.addresses == ["127.0.0.1", "::2", "::1"]
assert tcp3.ip_index == 2
# expiring entries should recalculate the pointer
now = Utils.now
tcp4 = tcp_class.new(origin, [Resolver::Entry.new("127.0.0.1", now + 1), Resolver::Entry.new("127.0.0.2", now + 4)], options)
assert tcp4.addresses == ["127.0.0.1", "127.0.0.2"]
assert tcp4.ip_index == 1
sleep(2)
assert tcp4.addresses?
assert tcp4.addresses == ["127.0.0.2"]
assert tcp4.ip_index.zero?
sleep(2)
assert !tcp4.addresses?
assert tcp4.ip_index == -1
end
end

View File

@ -16,21 +16,6 @@ module Requests
assert opts.oauth_session.token_endpoint.to_s == "#{server.origin}/token"
assert opts.oauth_session.token_endpoint_auth_method == "client_secret_basic"
assert opts.oauth_session.scope == %w[all]
assert opts.oauth_session.audience.nil?
# with audience
opts = HTTPX.plugin(:oauth).oauth_auth(
issuer: server.origin,
client_id: "CLIENT_ID", client_secret: "SECRET",
scope: "all",
audience: "audience"
).instance_variable_get(:@options)
assert opts.oauth_session.grant_type == "client_credentials"
assert opts.oauth_session.token_endpoint.to_s == "#{server.origin}/token"
assert opts.oauth_session.token_endpoint_auth_method == "client_secret_basic"
assert opts.oauth_session.scope == %w[all]
assert opts.oauth_session.audience == "audience"
# from options, pointing to refresh
opts = HTTPX.plugin(:oauth).oauth_auth(
@ -90,26 +75,6 @@ module Requests
end
end
def test_plugin_oauth_access_token_audience
with_oauth_metadata do |server|
http = HTTPX.plugin(:oauth).oauth_auth(
issuer: server.origin,
client_id: "CLIENT_ID", client_secret: "SECRET",
scope: "all",
)
http_aud = http.oauth_auth(
issuer: server.origin,
client_id: "CLIENT_ID", client_secret: "SECRET",
scope: "all", audience: "audience"
)
http_opts = http.with_access_token.instance_variable_get(:@options)
http_aud_opts = http_aud.with_access_token.instance_variable_get(:@options)
assert http_opts.oauth_session.access_token == "CLIENT-CREDS-AUTH"
assert http_aud_opts.oauth_session.access_token == "CLIENT-CREDS-AUTH-audience"
end
end
def test_plugin_oauth_client_credentials
with_oauth_metadata do |server|
session = HTTPX.plugin(:oauth).oauth_auth(

View File

@ -3,13 +3,14 @@
require_relative "test"
class ByIpCertServer < TestServer
USE_IPV6 = HTTPX::Options::DEFAULT_OPTIONS[:ip_families].size > 1
CERTS_DIR = File.expand_path("../ci/certs", __dir__)
def initialize
cert = OpenSSL::X509::Certificate.new(File.read(File.join(CERTS_DIR, "localhost-server.crt")))
key = OpenSSL::PKey.read(File.read(File.join(CERTS_DIR, "localhost-server.key")))
super(
:BindAddress => HTTPX::Resolver.supported_ip_families.size > 1 ? "::1" : "127.0.0.1",
:BindAddress => USE_IPV6 ? "::1" : "127.0.0.1",
:SSLEnable => true,
:SSLCertificate => cert,
:SSLPrivateKey => key,

View File

@ -26,30 +26,20 @@ class OAuthProviderServer < TestServer
end
res["content-type"] = "application/json"
token = +""
case body["grant_type"]
when "client_credentials"
token << "CLIENT-CREDS-AUTH" if user == "CLIENT_ID" && pass == "SECRET"
if (aud = body["audience"])
token << "-" << aud
if user == "CLIENT_ID" && pass == "SECRET"
res.body = JSON.dump({ "access_token" => "CLIENT-CREDS-AUTH", "expires_in" => 3600, "token_type" => "bearer" })
nil
end
res.body = JSON.dump({ "access_token" => token, "expires_in" => 3600, "token_type" => "bearer" })
when "refresh_token"
token << "REFRESH-TOKEN-AUTH" if user == "CLIENT_ID" && pass == "SECRET" && body["refresh_token"] == "REFRESH_TOKEN"
if (aud = body["audience"])
token << "-" << aud
if user == "CLIENT_ID" && pass == "SECRET" && body["refresh_token"] == "REFRESH_TOKEN"
res.body = JSON.dump({ "access_token" => "REFRESH-TOKEN-AUTH", "expires_in" => 3600, "token_type" => "bearer" })
nil
end
else
raise "unsupported"
end
raise "unsupported" if token.empty?
res.body = JSON.dump({ "access_token" => token, "expires_in" => 3600, "token_type" => "bearer" })
end
end