diff --git a/lib/httpx/adapters/webmock.rb b/lib/httpx/adapters/webmock.rb index ff1ede61..1adf14ed 100644 --- a/lib/httpx/adapters/webmock.rb +++ b/lib/httpx/adapters/webmock.rb @@ -60,7 +60,7 @@ module WebMock connection.once(:unmock_connection) do next unless connection.current_session == self - unless connection.addresses + unless connection.addresses? # reset Happy Eyeballs, fail early connection.sibling = nil diff --git a/lib/httpx/connection.rb b/lib/httpx/connection.rb index f13694f1..ee89bea2 100644 --- a/lib/httpx/connection.rb +++ b/lib/httpx/connection.rb @@ -122,6 +122,10 @@ module HTTPX @io && @io.addresses end + def addresses? + @io && @io.addresses? + end + def match?(uri, options) return false if !used? && (@state == :closing || @state == :closed) @@ -539,7 +543,7 @@ module HTTPX def send_request_to_parser(request) @inflight += 1 - request.peer_address = @io.ip + request.peer_address = @io.ip && @io.ip.address set_request_timeouts(request) parser.send(request) diff --git a/lib/httpx/io/tcp.rb b/lib/httpx/io/tcp.rb index eafe41d9..428d4e9a 100644 --- a/lib/httpx/io/tcp.rb +++ b/lib/httpx/io/tcp.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "resolv" -require "ipaddr" module HTTPX class TCP @@ -30,7 +29,8 @@ module HTTPX end raise Error, "Given IO objects do not match the request authority" unless @io - _, _, _, @ip = @io.addr + _, _, _, ip = @io.addr + @ip = Resolver::Entry.new(ip) @addresses << @ip @keep_open = true @state = :connected @@ -47,8 +47,6 @@ module HTTPX def add_addresses(addrs) return if addrs.empty? - addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) } - ip_index = @ip_index || (@addresses.size - 1) if addrs.first.ipv6? # should be the next in line @@ -59,6 +57,13 @@ module HTTPX end end + # eliminates expired entries and returns whether there are still any left. + def addresses? + @addresses.delete_if(&:expired?) + + @addresses.any? + end + def to_io @io.to_io end @@ -167,7 +172,7 @@ module HTTPX # do not mess with external sockets return false if @options.io - return true unless @addresses + return true if @addresses.empty? resolver_addresses = Resolver.nolookup_resolve(@hostname) diff --git a/lib/httpx/io/unix.rb b/lib/httpx/io/unix.rb index e0fb025d..7f2b0797 100644 --- a/lib/httpx/io/unix.rb +++ b/lib/httpx/io/unix.rb @@ -51,6 +51,11 @@ module HTTPX IO::WaitReadable end + # the path is always explicitly passed, so no point in resolving. + def addresses? + true + end + def expired? false end diff --git a/lib/httpx/options.rb b/lib/httpx/options.rb index ced44879..414354b6 100644 --- a/lib/httpx/options.rb +++ b/lib/httpx/options.rb @@ -210,7 +210,7 @@ module HTTPX end def option_addresses(value) - Array(value) + Array(value).map { |entry| Resolver::Entry.convert(entry) } end def option_ip_families(value) diff --git a/lib/httpx/plugins/ssrf_filter.rb b/lib/httpx/plugins/ssrf_filter.rb index 3c9933af..768cd2d0 100644 --- a/lib/httpx/plugins/ssrf_filter.rb +++ b/lib/httpx/plugins/ssrf_filter.rb @@ -129,8 +129,6 @@ module HTTPX end def addresses=(addrs) - addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) } - addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?)) raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty? diff --git a/lib/httpx/request/body.rb b/lib/httpx/request/body.rb index ce106f67..5c169371 100644 --- a/lib/httpx/request/body.rb +++ b/lib/httpx/request/body.rb @@ -127,8 +127,13 @@ module HTTPX # @type var body: bodyIO Transcoder::Body.encode(body) elsif (form = params.delete(:form)) - # @type var form: Transcoder::urlencoded_input - Transcoder::Form.encode(form) + if Transcoder::Multipart.multipart?(form) + # @type var form: Transcoder::multipart_input + Transcoder::Multipart.encode(form) + else + # @type var form: Transcoder::urlencoded_input + Transcoder::Form.encode(form) + end elsif (json = params.delete(:json)) # @type var body: _ToJson Transcoder::JSON.encode(json) diff --git a/lib/httpx/resolver.rb b/lib/httpx/resolver.rb index 170e06e8..abbadb90 100644 --- a/lib/httpx/resolver.rb +++ b/lib/httpx/resolver.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true require "resolv" -require "ipaddr" module HTTPX module Resolver RESOLVE_TIMEOUT = [2, 3].freeze + require "httpx/resolver/entry" require "httpx/resolver/resolver" require "httpx/resolver/system" require "httpx/resolver/native" @@ -39,16 +39,19 @@ module HTTPX ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname) end + # tries to convert +hostname+ into an IPAddr, returns nil otherwise. def ip_resolve(hostname) - [IPAddr.new(hostname)] + [Entry.new(hostname)] rescue ArgumentError end + # matches +hostname+ to entries in the hosts file, returns nil if none is + # found, or there is no hosts file. def system_resolve(hostname) ips = @system_resolver.getaddresses(hostname) return if ips.empty? - ips.map { |ip| IPAddr.new(ip) } + ips.map { |ip| Entry.new(ip) } rescue IOError end @@ -108,7 +111,7 @@ module HTTPX if (als = address["alias"]) lookup(als, lookups, ttl) else - IPAddr.new(address["data"]) + Entry.new(address["data"], address["TTL"]) end end.compact diff --git a/lib/httpx/resolver/entry.rb b/lib/httpx/resolver/entry.rb new file mode 100644 index 00000000..ebbf508d --- /dev/null +++ b/lib/httpx/resolver/entry.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require "ipaddr" + +module HTTPX + module Resolver + class Entry < SimpleDelegator + attr_reader :address + + def self.convert(address) + new(address, rescue_on_convert: true) + end + + def initialize(address, expires_in = Float::INFINITY, rescue_on_convert: false) + @expires_in = expires_in + @address = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s) + super(@address) + rescue IPAddr::InvalidAddressError + raise unless rescue_on_convert + + @address = address.to_s + super(@address) + end + + def expired? + @expires_in < Utils.now + end + end + end +end diff --git a/lib/httpx/resolver/native.rb b/lib/httpx/resolver/native.rb index 7699f19b..ed50e84e 100644 --- a/lib/httpx/resolver/native.rb +++ b/lib/httpx/resolver/native.rb @@ -450,7 +450,7 @@ module HTTPX when :tcp log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." } origin = URI("tcp://#{ip}:#{port}") - TCP.new(origin, [ip], @options) + TCP.new(origin, [Resolver::Entry.new(ip)], @options) end end diff --git a/lib/httpx/resolver/resolver.rb b/lib/httpx/resolver/resolver.rb index ce9d7f30..b676cc16 100644 --- a/lib/httpx/resolver/resolver.rb +++ b/lib/httpx/resolver/resolver.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "resolv" -require "ipaddr" module HTTPX # Base class for all internal internet name resolvers. It handles basic blocks @@ -69,9 +68,7 @@ module HTTPX end def emit_addresses(connection, family, addresses, early_resolve = false) - addresses.map! do |address| - address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s) - end + addresses.map! { |address| address.is_a?(Resolver::Entry) ? address : Resolver::Entry.new(address) } # double emission check, but allow early resolution to work return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses) diff --git a/lib/httpx/session.rb b/lib/httpx/session.rb index 333bc87f..bd8a29e4 100644 --- a/lib/httpx/session.rb +++ b/lib/httpx/session.rb @@ -370,7 +370,7 @@ module HTTPX end def resolve_connection(connection, selector) - if connection.addresses || connection.open? + if connection.addresses? || connection.open? # # there are two cases in which we want to activate initialization of # connection immediately: diff --git a/lib/httpx/transcoder/form.rb b/lib/httpx/transcoder/form.rb index 968ebc30..a8d306f0 100644 --- a/lib/httpx/transcoder/form.rb +++ b/lib/httpx/transcoder/form.rb @@ -48,11 +48,7 @@ module HTTPX end def encode(form) - if multipart?(form) - Multipart::Encoder.new(form) - else - Encoder.new(form) - end + Encoder.new(form) end def decode(response) @@ -67,14 +63,6 @@ module HTTPX raise Error, "invalid form mime type (#{content_type})" end end - - def multipart?(data) - data.any? do |_, v| - Multipart::MULTIPART_VALUE_COND.call(v) || - (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) || - (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) }) - end - end end end end diff --git a/lib/httpx/transcoder/multipart.rb b/lib/httpx/transcoder/multipart.rb index e7aa20d2..035c6c12 100644 --- a/lib/httpx/transcoder/multipart.rb +++ b/lib/httpx/transcoder/multipart.rb @@ -13,5 +13,19 @@ module HTTPX::Transcoder value.key?(:body) && (value.key?(:filename) || value.key?(:content_type))) end + + module_function + + def multipart?(form_data) + form_data.any? do |_, v| + Multipart::MULTIPART_VALUE_COND.call(v) || + (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) || + (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) }) + end + end + + def encode(form_data) + Encoder.new(form_data) + end end end diff --git a/sig/connection.rbs b/sig/connection.rbs index 235ac5f9..33f6dd73 100644 --- a/sig/connection.rbs +++ b/sig/connection.rbs @@ -50,14 +50,16 @@ module HTTPX @main_sibling: bool - def addresses: () -> Array[ipaddr]? + def addresses: () -> Array[Resolver::Entry]? def peer: () -> URI::Generic - def addresses=: (Array[ipaddr] addresses) -> void + def addresses=: (Array[Resolver::Entry] addresses) -> void def send: (Request request) -> void + def addresses?: () -> boolish + def match?: (URI::Generic uri, Options options) -> bool def expired?: () -> boolish @@ -140,7 +142,7 @@ module HTTPX def build_altsvc_connection: (URI::Generic alt_origin, String origin, Hash[String, String] alt_params) -> void - def build_socket: (?Array[ipaddr]? addrs) -> (TCP | SSL | UNIX) + def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX) def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void diff --git a/sig/io/ssl.rbs b/sig/io/ssl.rbs index 41c7ac33..1786e3dd 100644 --- a/sig/io/ssl.rbs +++ b/sig/io/ssl.rbs @@ -12,7 +12,7 @@ module HTTPX attr_writer ssl_session: OpenSSL::SSL::Session? # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed - # def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) ?{ (self) -> void } -> void + # def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, options options) ?{ (self) -> void } -> void def session_new_cb: { (OpenSSL::SSL::Session sess) -> void } -> void def can_verify_peer?: () -> bool diff --git a/sig/io/tcp.rbs b/sig/io/tcp.rbs index 5b94c58d..d7fb144d 100644 --- a/sig/io/tcp.rbs +++ b/sig/io/tcp.rbs @@ -2,11 +2,11 @@ module HTTPX class TCP include Loggable - attr_reader ip: ipaddr? + attr_reader ip: Resolver::Entry? attr_reader port: Integer - attr_reader addresses: Array[ipaddr] + attr_reader addresses: Array[Resolver::Entry] attr_reader state: Symbol @@ -27,9 +27,11 @@ module HTTPX @ip_index: Integer # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed - def initialize: (URI::Generic origin, Array[ipaddr]? addresses, Options options) ?{ (instance) -> void } -> void + def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, Options options) ?{ (instance) -> void } -> void - def add_addresses: (Array[ipaddr] addrs) -> void + def add_addresses: (Array[Resolver::Entry] addrs) -> void + + def addresses?: () -> bool def to_io: () -> IO diff --git a/sig/options.rbs b/sig/options.rbs index 562b3375..515dc7a4 100644 --- a/sig/options.rbs +++ b/sig/options.rbs @@ -53,7 +53,7 @@ module HTTPX attr_reader transport: io_type | nil # addresses - attr_reader addresses: Array[ipaddr]? + attr_reader addresses: Array[Resolver::Entry]? # supported_compression_formats attr_reader supported_compression_formats: Array[String] diff --git a/sig/plugins/ssrf_filter.rbs b/sig/plugins/ssrf_filter.rbs index 9e0d9382..0a816498 100644 --- a/sig/plugins/ssrf_filter.rbs +++ b/sig/plugins/ssrf_filter.rbs @@ -8,7 +8,7 @@ module HTTPX IPV6_BLACKLIST: Array[[IPAddr, IPAddr]] - def self?.unsafe_ip_address?: (IPAddr) -> bool + def self?.unsafe_ip_address?: (Resolver::Entry ipaddr) -> bool interface _Options def allowed_schemes: () -> Array[String] diff --git a/sig/request.rbs b/sig/request.rbs index 46d70961..24fd47d5 100644 --- a/sig/request.rbs +++ b/sig/request.rbs @@ -17,7 +17,7 @@ module HTTPX attr_reader drain_error: StandardError? attr_reader active_timeouts: Array[Symbol] - attr_accessor peer_address: ipaddr? + attr_accessor peer_address: (String | IPAddr)? attr_writer persistent: bool diff --git a/sig/resolver.rbs b/sig/resolver.rbs index 158c3d58..40708ad5 100644 --- a/sig/resolver.rbs +++ b/sig/resolver.rbs @@ -1,5 +1,5 @@ module HTTPX - type ipaddr = IPAddr | String + type ipaddr = String | IPAddr | Resolv::IPv4 | Resolver::Entry module Resolver type dns_resource = singleton(Resolv::DNS::Resource) @@ -17,21 +17,21 @@ module HTTPX type dns_decoding_response = [:ok, Array[dns_result]] | [:decode_error, Resolv::DNS::DecodeError] | [:dns_error, Integer] | Symbol - def self?.nolookup_resolve: (String hostname) -> Array[IPAddr]? + def self?.nolookup_resolve: (String hostname) -> Array[Entry]? - def self?.ip_resolve: (String hostname) -> Array[IPAddr]? + def self?.ip_resolve: (String hostname) -> Array[Entry]? - def self?.system_resolve: (String hostname) -> Array[IPAddr]? + def self?.system_resolve: (String hostname) -> Array[Entry]? def self?.resolver_for: (Symbol | singleton(Resolver) resolver_type, Options options) -> singleton(Resolver) - def self?.cached_lookup: (String hostname) -> Array[IPAddr]? + def self?.cached_lookup: (String hostname) -> Array[Entry]? def self?.cached_lookup_set: (String hostname, ip_family family, Array[dns_result] addresses) -> void - def self?.cached_lookup_evict: (String hostname, ipaddr ip) -> void + def self?.cached_lookup_evict: (String hostname, _ToS ip) -> void - def self?.lookup: (String hostname, Hash[String, Array[dns_result]] lookups, Numeric ttl) -> Array[IPAddr]? + def self?.lookup: (String hostname, Hash[String, Array[dns_result]] lookups, Numeric ttl) -> Array[Entry]? def self?.generate_id: () -> Integer diff --git a/sig/resolver/entry.rbs b/sig/resolver/entry.rbs new file mode 100644 index 00000000..d8c3b86b --- /dev/null +++ b/sig/resolver/entry.rbs @@ -0,0 +1,13 @@ +module HTTPX + class Resolver::Entry + attr_reader address: IPAddr | String + + @expires_in: Numeric + + def initialize: (IPAddr | _ToS address, ?Numeric expires_in, ?rescue_on_convert: boolish) -> void + + def expired?: () -> bool + + def self.convert: (IPAddr | _ToS address) -> Resolver::Entry + end +end \ No newline at end of file diff --git a/sig/resolver/resolver.rbs b/sig/resolver/resolver.rbs index 2799156e..ec4c9af1 100644 --- a/sig/resolver/resolver.rbs +++ b/sig/resolver/resolver.rbs @@ -35,7 +35,7 @@ module HTTPX def each_connection: () { (Connection connection) -> void } -> void - def emit_addresses: (Connection connection, ip_family family, Array[IPAddr], ?bool early_resolve) -> void + def emit_addresses: (Connection connection, ip_family family, Array[ipaddr], ?bool early_resolve) -> void def self.multi?: () -> bool @@ -43,7 +43,7 @@ module HTTPX def resolve: (?Connection connection, ?String hostname) -> void - def emit_resolved_connection: (Connection connection, Array[IPAddr] addresses, bool early_resolve) -> void + def emit_resolved_connection: (Connection connection, Array[ipaddr] addresses, bool early_resolve) -> void def initialize: (ip_family family, Options options) -> void diff --git a/sig/response.rbs b/sig/response.rbs index 069ca7cd..a6d1593f 100644 --- a/sig/response.rbs +++ b/sig/response.rbs @@ -35,7 +35,7 @@ module HTTPX def uri: () -> URI::Generic - def peer_address: () -> ipaddr? + def peer_address: () -> (String | IPAddr)? def merge_headers: (_Each[[String, headers_value]]) -> void @@ -90,7 +90,7 @@ module HTTPX def uri: () -> URI::Generic - def peer_address: () -> ipaddr? + def peer_address: () -> (String | IPAddr)? def close: () -> void diff --git a/sig/transcoder/form.rbs b/sig/transcoder/form.rbs index 9410cd78..da43fe0a 100644 --- a/sig/transcoder/form.rbs +++ b/sig/transcoder/form.rbs @@ -3,14 +3,14 @@ module HTTPX::Transcoder type form_nested_value = form_value | _ToAry[form_value] | _ToHash[string, form_value] - type urlencoded_input = Enumerable[[_ToS, form_nested_value | Multipart::multipart_nested_value]] + type urlencoded_input = Enumerable[[_ToS, form_nested_value]] module Form PARAM_DEPTH_LIMIT: Integer - def self?.encode: (urlencoded_input form) -> (Encoder | Multipart::Encoder) + def self?.encode: (urlencoded_input form) -> Encoder + def self?.decode: (HTTPX::Response response) -> _Decoder - def self?.multipart?: (form_nested_value | Multipart::multipart_nested_value data) -> bool class Encoder extend Forwardable diff --git a/sig/transcoder/multipart.rbs b/sig/transcoder/multipart.rbs index 50bd019c..2bb5f59d 100644 --- a/sig/transcoder/multipart.rbs +++ b/sig/transcoder/multipart.rbs @@ -16,6 +16,12 @@ module HTTPX type multipart_nested_value = multipart_value | _ToAry[multipart_value] | _ToHash[string, multipart_value] + type multipart_input = Enumerable[[_ToS, Multipart::multipart_nested_value]] + + def self?.encode: (multipart_input form_data) -> Multipart::Encoder + + def self?.multipart?: (form_nested_value | multipart_nested_value form_data) -> bool + class Encoder @boundary: String @part_index: Integer @@ -36,9 +42,9 @@ module HTTPX private - def to_parts: (Enumerable[[Symbol | string, multipart_nested_value]] multipart_data) -> Array[_Reader] + def to_parts: (multipart_input multipart_data) -> Array[_Reader] - def initialize: (Enumerable[[Symbol | string, multipart_nested_value]] multipart_data) -> untyped + def initialize: (multipart_input multipart_data) -> untyped def header_part: (String key, String content_type, String? filename) -> StringIO