Merge branch 'improv' into 'master'

sig improvements

See merge request os85/httpx!390
This commit is contained in:
HoneyryderChuck 2025-05-13 15:18:50 +00:00
commit cf19fe5221
36 changed files with 232 additions and 70 deletions

View File

@ -11,8 +11,8 @@ module HTTPX
MAX_CONCURRENT_REQUESTS = ::HTTP2::DEFAULT_MAX_CONCURRENT_STREAMS
class Error < Error
def initialize(id, code)
super("stream #{id} closed with error: #{code}")
def initialize(id, error)
super("stream #{id} closed with error: #{error}")
end
end

View File

@ -9,7 +9,8 @@ module HTTPX
# rubocop:disable Style/MutableConstant
TLS_OPTIONS = { alpn_protocols: %w[h2 http/1.1].freeze }
# https://github.com/jruby/jruby-openssl/issues/284
TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby"
# TODO: remove when dropping support for jruby-openssl < 0.15.4
TLS_OPTIONS[:verify_hostname] = true if RUBY_ENGINE == "jruby" && JOpenSSL::VERSION < "0.15.4"
# rubocop:enable Style/MutableConstant
TLS_OPTIONS.freeze

View File

@ -147,13 +147,18 @@ module HTTPX
end
def freeze
super
@origin.freeze
@base_path.freeze
@timeout.freeze
@headers.freeze
@addresses.freeze
@supported_compression_formats.freeze
@ssl.freeze
@http2_settings.freeze
@pool_options.freeze
@resolver_options.freeze
@ip_families.freeze
super
end
def option_origin(value)
@ -226,17 +231,42 @@ module HTTPX
Array(value)
end
# number options
%i[
max_concurrent_requests max_requests window_size buffer_size
body_threshold_size debug_level
].each do |option|
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
# converts +v+ into an Integer before setting the +#{option}+ option.
def option_#{option}(value) # def option_max_requests(v)
value = Integer(value) unless value.infinite?
raise TypeError, ":#{option} must be positive" unless value.positive? # raise TypeError, ":max_requests must be positive" unless value.positive?
value
end
OUT
end
# hashable options
%i[ssl http2_settings resolver_options pool_options].each do |option|
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
# converts +v+ into an Hash before setting the +#{option}+ option.
def option_#{option}(value) # def option_ssl(v)
Hash[value]
end
OUT
end
%i[
ssl http2_settings
request_class response_class headers_class request_body_class
response_body_class connection_class options_class
pool_class pool_options
io fallback_protocol debug debug_level resolver_class resolver_options
io fallback_protocol debug resolver_class
compress_request_body decompress_response_body
persistent close_on_fork
].each do |method_name|
class_eval(<<-OUT, __FILE__, __LINE__ + 1)
# sets +v+ as the value of #{method_name}
# sets +v+ as the value of the +#{method_name}+ option
def option_#{method_name}(v); v; end # def option_smth(v); v; end
OUT
end

View File

@ -59,8 +59,6 @@ module HTTPX
return @cookies.each(&blk) unless uri
uri = URI(uri)
now = Time.now
tpath = uri.path

View File

@ -83,7 +83,7 @@ module HTTPX
scanner.skip(RE_WSP)
name, value = scan_name_value(scanner, true)
value = nil if name.empty?
value = nil if name && name.empty?
attrs = {}
@ -98,15 +98,18 @@ module HTTPX
aname, avalue = scan_name_value(scanner, true)
next if aname.empty? || value.nil?
next if (aname.nil? || aname.empty?) || value.nil?
aname.downcase!
case aname
when "expires"
next unless avalue
# RFC 6265 5.2.1
(avalue &&= Time.parse(avalue)) || next
(avalue = Time.parse(avalue)) || next
when "max-age"
next unless avalue
# RFC 6265 5.2.2
next unless /\A-?\d+\z/.match?(avalue)
@ -119,7 +122,7 @@ module HTTPX
# RFC 6265 5.2.4
# A relative path must be ignored rather than normalizing it
# to "/".
next unless avalue.start_with?("/")
next unless avalue && avalue.start_with?("/")
when "secure", "httponly"
# RFC 6265 5.2.5, 5.2.6
avalue = true

View File

@ -155,6 +155,9 @@ module HTTPX
@pipe_read.close
@closed = true
end
# noop (the owner connection will take of it)
def handle_socket_timeout(interval); end
end
class << self

View File

@ -92,8 +92,8 @@ module HTTPX
end
ips = entries.flat_map do |address|
if address.key?("alias")
lookup(address["alias"], lookups, ttl)
if (als = address["alias"])
lookup(als, lookups, ttl)
else
IPAddr.new(address["data"])
end

View File

@ -6,6 +6,10 @@ require "forwardable"
require "httpx/base64"
module HTTPX
# Implementation of a DoH name resolver (https://www.youtube.com/watch?v=unMXvnY2FNM).
# It wraps an HTTPX::Connection object which integrates with the main session in the
# same manner as other performed HTTP requests.
#
class Resolver::HTTPS < Resolver::Resolver
extend Forwardable
using URIExtensions
@ -26,14 +30,13 @@ module HTTPX
use_get: false,
}.freeze
def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?
def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close, :terminate, :inflight?, :handle_socket_timeout
def initialize(_, options)
super
@resolver_options = DEFAULTS.merge(@options.resolver_options)
@queries = {}
@requests = {}
@connections = []
@uri = URI(@resolver_options[:uri])
@uri_addresses = nil
@resolver = Resolv::DNS.new
@ -74,7 +77,11 @@ module HTTPX
private
def resolve(connection = @connections.first, hostname = nil)
def resolve(connection = nil, hostname = nil)
@connections.shift until @connections.empty? || @connections.first.state != :closed
connection ||= @connections.first
return unless connection
hostname ||= @queries.key(connection)

View File

@ -4,6 +4,9 @@ require "forwardable"
require "resolv"
module HTTPX
# Implements a pure ruby name resolver, which abides by the Selectable API.
# It delegates DNS payload encoding/decoding to the +resolv+ stlid gem.
#
class Resolver::Native < Resolver::Resolver
extend Forwardable
using URIExtensions
@ -34,7 +37,6 @@ module HTTPX
@search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
@_timeouts = Array(@resolver_options[:timeouts])
@timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
@connections = []
@name = nil
@queries = {}
@read_buffer = "".b
@ -505,7 +507,7 @@ module HTTPX
end
while (connection = @connections.shift)
emit_resolve_error(connection, host, error)
emit_resolve_error(connection, connection.peer.host, error)
end
end
end

View File

@ -4,6 +4,9 @@ require "resolv"
require "ipaddr"
module HTTPX
# Base class for all internal internet name resolvers. It handles basic blocks
# from the Selectable API.
#
class Resolver::Resolver
include Callbacks
include Loggable
@ -36,6 +39,7 @@ module HTTPX
@family = family
@record_type = RECORD_TYPES[family]
@options = options
@connections = []
set_resolver_callbacks
end

View File

@ -3,6 +3,15 @@
require "resolv"
module HTTPX
# Implementation of a synchronous name resolver which relies on the system resolver,
# which is lib'c getaddrinfo function (abstracted in ruby via Addrinfo.getaddrinfo).
#
# Its main advantage is relying on the reference implementation for name resolution
# across most/all OSs which deploy ruby (it's what TCPSocket also uses), its main
# disadvantage is the inability to set timeouts / check socket for readiness events,
# hence why it relies on using the Timeout module, which poses a lot of problems for
# the selector loop, specially when network is unstable.
#
class Resolver::System < Resolver::Resolver
using URIExtensions
@ -23,14 +32,13 @@ module HTTPX
attr_reader :state
def initialize(options)
super(nil, options)
super(0, options)
@resolver_options = @options.resolver_options
resolv_options = @resolver_options.dup
timeouts = resolv_options.delete(:timeouts) || Resolver::RESOLVE_TIMEOUT
@_timeouts = Array(timeouts)
@timeouts = Hash.new { |tims, host| tims[host] = @_timeouts.dup }
resolv_options.delete(:cache)
@connections = []
@queries = []
@ips = []
@pipe_mutex = Thread::Mutex.new
@ -100,7 +108,14 @@ module HTTPX
def handle_socket_timeout(interval)
error = HTTPX::ResolveTimeoutError.new(interval, "timed out while waiting on select")
error.set_backtrace(caller)
on_error(error)
@queries.each do |host, connection|
@connections.delete(connection)
emit_resolve_error(connection, host, error)
end
while (connection = @connections.shift)
emit_resolve_error(connection, connection.peer.host, error)
end
end
private
@ -131,19 +146,22 @@ module HTTPX
case event
when DONE
*pair, addrs = @pipe_mutex.synchronize { @ips.pop }
@queries.delete(pair)
_, connection = pair
@connections.delete(connection)
if pair
@queries.delete(pair)
family, connection = pair
@connections.delete(connection)
family, connection = pair
catch(:coalesced) { emit_addresses(connection, family, addrs) }
catch(:coalesced) { emit_addresses(connection, family, addrs) }
end
when ERROR
*pair, error = @pipe_mutex.synchronize { @ips.pop }
@queries.delete(pair)
@connections.delete(connection)
if pair && error
@queries.delete(pair)
@connections.delete(connection)
_, connection = pair
emit_resolve_error(connection, connection.peer.host, error)
_, connection = pair
emit_resolve_error(connection, connection.peer.host, error)
end
end
end
@ -152,11 +170,16 @@ module HTTPX
resolve
end
def resolve(connection = @connections.first)
def resolve(connection = nil, hostname = nil)
@connections.shift until @connections.empty? || @connections.first.state != :closed
connection ||= @connections.first
raise Error, "no URI to resolve" unless connection
return unless @queries.empty?
hostname = connection.peer.host
hostname ||= connection.peer.host
scheme = connection.origin.scheme
log do
"resolver: resolve IDN #{connection.peer.non_ascii_hostname} as #{hostname}"

View File

@ -14,7 +14,7 @@ module HTTPX
def capacity: () -> Integer
# delegated
def <<: (string data) -> String
def <<: (String data) -> String
def empty?: () -> bool
def bytesize: () -> (Integer | Float)
def clear: () -> void

View File

@ -21,8 +21,9 @@ module HTTPX
| (:cookies, ?options) -> Plugins::sessionCookies
| (:expect, ?options) -> Session
| (:follow_redirects, ?options) -> Plugins::sessionFollowRedirects
| (:upgrade, ?options) -> Session
| (:h2c, ?options) -> Session
| (:upgrade, ?options) -> Plugins::sessionUpgrade
| (:h2c, ?options) -> Plugins::sessionUpgrade
| (:h2, ?options) -> Plugins::sessionUpgrade
| (:persistent, ?options) -> Plugins::sessionPersistent
| (:proxy, ?options) -> (Plugins::sessionProxy & Plugins::httpProxy)
| (:push_promise, ?options) -> Plugins::sessionPushPromise

View File

@ -20,6 +20,7 @@ module HTTPX
attr_reader type: io_type
attr_reader io: TCP | SSL | UNIX | nil
attr_reader origin: http_uri
attr_reader origins: Array[String]
attr_reader state: Symbol
@ -39,7 +40,6 @@ module HTTPX
@keep_alive_timeout: Numeric?
@timeout: Numeric?
@current_timeout: Numeric?
@io: TCP | SSL | UNIX
@parser: Object & _Parser
@connected_at: Float
@response_received_at: Float

View File

@ -16,6 +16,8 @@ module HTTPX
@drains: Hash[Request, String]
@pings: Array[String]
@buffer: Buffer
@handshake_completed: bool
@wait_for_handshake: bool
def interests: () -> io_interests?
@ -96,12 +98,15 @@ module HTTPX
def on_pong: (string ping) -> void
class Error < ::HTTPX::Error
def initialize: (Integer id, Symbol | StandardError error) -> void
end
class GoawayError < Error
def initialize: () -> void
end
class PingError < Error
def initialize: () -> void
end
end
end

View File

@ -13,6 +13,7 @@ module HTTPX
KEEP_ALIVE_TIMEOUT: Integer
SETTINGS_TIMEOUT: Integer
CLOSE_HANDSHAKE_TIMEOUT: Integer
SET_TEMPORARY_NAME: ^(Module mod, ?Symbol pl) -> void
DEFAULT_OPTIONS: Hash[Symbol, untyped]
REQUEST_BODY_IVARS: Array[Symbol]
@ -89,6 +90,8 @@ module HTTPX
attr_reader response_body_class: singleton(Response::Body)
attr_reader options_class: singleton(Options)
attr_reader resolver_class: Symbol | Class
attr_reader ssl: Hash[Symbol, untyped]

View File

@ -1,6 +1,8 @@
module HTTPX
module Plugins
module Cookies
type cookie_attributes = Hash[Symbol | String, top]
type jar = Jar | _Each[Jar::cookie]
interface _CookieOptions

View File

@ -1,7 +1,5 @@
module HTTPX
module Plugins::Cookies
type cookie_attributes = Hash[Symbol | String, top]
class Cookie
include Comparable
@ -33,7 +31,7 @@ module HTTPX
def cookie_value: () -> String
alias to_s cookie_value
def valid_for_uri?: (uri) -> bool
def valid_for_uri?: (http_uri uri) -> bool
def self.new: (Cookie) -> instance
| (cookie_attributes) -> instance

View File

@ -11,12 +11,12 @@ module HTTPX
def add: (Cookie name, ?String path) -> void
def []: (uri) -> Array[Cookie]
def []: (http_uri) -> Array[Cookie]
def each: (?uri?) { (Cookie) -> void } -> void
| (?uri?) -> Enumerable[Cookie]
def each: (?http_uri?) { (Cookie) -> void } -> void
| (?http_uri?) -> Enumerable[Cookie]
def merge: (_Each[cookie] cookies) -> instance
def merge: (_Each[cookie] cookies) -> self
private

View File

@ -0,0 +1,22 @@
module HTTPX
module Plugins::Cookies
module SetCookieParser
RE_WSP: Regexp
RE_NAME: Regexp
RE_BAD_CHAR: Regexp
RE_COOKIE_COMMA: Regexp
def self?.call: (String set_cookie) { (String name, String value, cookie_attributes attrs) -> void } -> void
def self?.scan_dquoted: (StringScanner scanner) -> String
def self?.scan_value: (StringScanner scanner, ?bool comma_as_separator) -> String
def self?.scan_name_value: (StringScanner scanner, ?bool comma_as_separator) -> [String?, String?]
end
end
end

View File

@ -49,6 +49,10 @@ module HTTPX
def self.extra_options: (Options) -> (Options & _ProxyOptions)
module ConnectionMethods
@proxy_uri: generic_uri
end
module InstanceMethods
@__proxy_uris: Array[generic_uri]

View File

@ -16,6 +16,9 @@ module HTTPX
def __http_on_connect: (top, Response) -> void
end
module ProxyParser
end
class ConnectRequest < Request
def initialize: (generic_uri uri, Options options) -> void
end

View File

@ -16,6 +16,9 @@ module HTTPX
def stream: () -> StreamResponse?
end
module ResponseBodyMethods
@stream: StreamResponse?
end
end
type sessionStream = Session & Stream::InstanceMethods

View File

@ -13,6 +13,9 @@ module HTTPX
def self.extra_options: (Options) -> (Options & _UpgradeOptions)
module InstanceMethods
end
module ConnectionMethods
attr_reader upgrade_protocol: String?
attr_reader hijacked: boolish
@ -20,5 +23,7 @@ module HTTPX
def hijack_io: () -> void
end
end
type sessionUpgrade = Session & Upgrade::InstanceMethods
end
end

View File

@ -0,0 +1,9 @@
module HTTPX
module Plugins
module H2
module ConnectionMethods
def upgrade_to_h2: () -> void
end
end
end
end

5
sig/punycode.rbs Normal file
View File

@ -0,0 +1,5 @@
module HTTPX
module Punycode
def self?.encode_hostname: (String) -> String
end
end

View File

@ -2,22 +2,26 @@ module HTTPX
type ipaddr = IPAddr | String
module Resolver
RESOLVE_TIMEOUT: Array[Integer]
@lookup_mutex: Thread::Mutex
type dns_resource = singleton(Resolv::DNS::Resource)
type dns_result = { "name" => String, "TTL" => Numeric, "alias" => String }
| { "name" => String, "TTL" => Numeric, "data" => String }
RESOLVE_TIMEOUT: Array[Integer]
self.@lookup_mutex: Thread::Mutex
self.@lookups: Hash[String, Array[dns_result]]
self.@identifier_mutex: Thread::Mutex
self.@identifier: Integer
self.@system_resolver: Resolv::Hosts
type dns_decoding_response = [:ok, Array[dns_result]] | [:decode_error, Resolv::DNS::DecodeError] | [:dns_error, Integer] | Symbol
def nolookup_resolve: (String hostname) -> Array[IPAddr]
def self?.nolookup_resolve: (String hostname) -> Array[IPAddr]?
def ip_resolve: (String hostname) -> Array[IPAddr]?
def self?.ip_resolve: (String hostname) -> Array[IPAddr]?
def system_resolve: (String hostname) -> Array[IPAddr]?
def self?.system_resolve: (String hostname) -> Array[IPAddr]?
def self?.resolver_for: (:native resolver_type) -> singleton(Native) |
(:system resolver_type) -> singleton(System) |

View File

@ -1,6 +1,8 @@
module HTTPX
module Resolver
class HTTPS < Resolver
extend Forwardable
NAMESERVER: String
DEFAULTS: Hash[Symbol, untyped]
@ -9,6 +11,7 @@ module HTTPX
attr_reader family: ip_family
@options: Options
@queries: Hash[String, Connection]
@requests: Hash[Request, String]
@connections: Array[Connection]
@uri: http_uri
@ -26,8 +29,6 @@ module HTTPX
def resolver_connection: () -> Connection
def resolve: (?Connection connection, ?String? hostname) -> void
def on_response: (Request, response) -> void
def parse: (Request request, Response response) -> void

View File

@ -16,6 +16,7 @@ module HTTPX
@search: Array[String]
@_timeouts: Array[Numeric]
@timeouts: Hash[String, Array[Numeric]]
@queries: Hash[String, Connection]
@connections: Array[Connection]
@read_buffer: String
@write_buffer: Buffer
@ -54,8 +55,6 @@ module HTTPX
def parse: (String) -> void
def resolve: (?Connection connection, ?String hostname) -> void
def generate_candidates: (String) -> Array[String]
def build_socket: () -> (UDP | TCP)

View File

@ -4,7 +4,11 @@ module HTTPX
include Callbacks
include Loggable
RECORD_TYPES: Hash[Integer, singleton(Resolv::DNS::Resource)]
include _Selectable
RECORD_TYPES: Hash[ip_family, singleton(Resolv::DNS::Resource)]
FAMILY_TYPES: Hash[singleton(Resolv::DNS::Resource), String]
attr_reader family: ip_family
@ -18,8 +22,8 @@ module HTTPX
@record_type: singleton(Resolv::DNS::Resource)
@resolver_options: Hash[Symbol, untyped]
@queries: Hash[String, Connection]
@system_resolver: Resolv::Hosts
@connections: Array[Connection]
def close: () -> void
@ -33,11 +37,15 @@ module HTTPX
def emit_addresses: (Connection connection, ip_family family, Array[IPAddr], ?bool early_resolve) -> void
def self.multi?: () -> bool
private
def resolve: (?Connection connection, ?String hostname) -> void
def emit_resolved_connection: (Connection connection, Array[IPAddr] addresses, bool early_resolve) -> void
def initialize: (ip_family? family, Options options) -> void
def initialize: (ip_family family, Options options) -> void
def early_resolve: (Connection connection, ?hostname: String) -> bool

View File

@ -2,16 +2,33 @@ module HTTPX
module Resolver
class System < Resolver
RESOLV_ERRORS: Array[singleton(StandardError)] # ResolvError
DONE: 1
ERROR: 2
@resolver: Resolv::DNS
@_timeouts: Array[Numeric]
@timeouts: Hash[String, Array[Numeric]]
@queries: Array[[ip_family, Connection]]
@ips: Array[[ip_family, Connection, (Array[Addrinfo] | StandardError)]]
@pipe_mutex: Thread::Mutex
@pipe_read: ::IO
@pipe_write: ::IO
attr_reader family: nil
attr_reader state: Symbol
def <<: (Connection) -> void
private
def initialize: (options options) -> void
def transition: (Symbol nextstate) -> void
def consume: () -> void
def async_resolve: (Connection connection, String hostname, String scheme) -> void
def __addrinfo_resolve: (String host, String scheme) -> Array[Addrinfo]
def initialize: (Options options) -> void
end
end
end

View File

@ -9,6 +9,8 @@ module HTTPX
def interests: () -> io_interests?
def timeout: () -> Numeric?
def handle_socket_timeout: (Numeric interval) -> void
end
class Selector

View File

@ -96,5 +96,5 @@ module HTTPX
end
end
OriginalSession: singleton(Session)
S: singleton(Session)
end

View File

@ -47,17 +47,17 @@ class CookieJarTest < Minitest::Test
domain_jar = HTTPX::Plugins::Cookies::Jar.new
domain_jar.parse(%(a=b; Path=/; Domain=.google.com))
assert domain_jar[jar_cookies_uri].empty?
assert !domain_jar["http://www.google.com/"].empty?
assert !domain_jar[URI("http://www.google.com/")].empty?
ipv4_domain_jar = HTTPX::Plugins::Cookies::Jar.new
ipv4_domain_jar.parse(%(a=b; Path=/; Domain=137.1.0.12))
assert ipv4_domain_jar["http://www.google.com/"].empty?
assert !ipv4_domain_jar["http://137.1.0.12/"].empty?
assert ipv4_domain_jar[URI("http://www.google.com/")].empty?
assert !ipv4_domain_jar[URI("http://137.1.0.12/")].empty?
ipv6_domain_jar = HTTPX::Plugins::Cookies::Jar.new
ipv6_domain_jar.parse(%(a=b; Path=/; Domain=[fe80::1]))
assert ipv6_domain_jar["http://www.google.com/"].empty?
assert !ipv6_domain_jar["http://[fe80::1]/"].empty?
assert ipv6_domain_jar[URI("http://www.google.com/")].empty?
assert !ipv6_domain_jar[URI("http://[fe80::1]/")].empty?
# Test duplicate
dup_jar = HTTPX::Plugins::Cookies::Jar.new
@ -110,6 +110,6 @@ class CookieJarTest < Minitest::Test
private
def jar_cookies_uri(path = "/cookies", scheme: "http")
"#{scheme}://example.com#{path}"
URI("#{scheme}://example.com#{path}")
end
end

View File

@ -66,7 +66,7 @@ if [[ "$RUBY_ENGINE" = "ruby" ]] && [[ ${RUBY_VERSION:0:3} = "3.4" ]] && [[ ! $R
export RUBYOPT="$RUBYOPT -rbundler/setup -rrbs/test/setup"
export RBS_TEST_RAISE=true
export RBS_TEST_LOGLEVEL=error
export RBS_TEST_OPT="-Isig -rset -rforwardable -ruri -rjson -ripaddr -rpathname -rtime -rtimeout -rresolv -rsocket -ropenssl -rbase64 -rzlib -rcgi -rdigest -rhttp-2"
export RBS_TEST_OPT="-Isig -rset -rforwardable -ruri -rjson -ripaddr -rpathname -rtime -rtimeout -rresolv -rsocket -ropenssl -rbase64 -rzlib -rcgi -rdigest -rstrscan -rhttp-2"
export RBS_TEST_TARGET="HTTP*"
fi

View File

@ -130,7 +130,7 @@ module Requests
end
def cookies_set_uri(cookies)
build_uri("/cookies/set?#{URI.encode_www_form(cookies)}")
URI(build_uri("/cookies/set?#{URI.encode_www_form(cookies)}"))
end
def verify_cookies(jar, cookies)