Merge branch 'issue-322' into 'master'

drop ip addresses from persistent connections which ran out of TTL

Closes #322

See merge request os85/httpx!404

lib/httpx/adapters/webmock.rb

@@ -60,7 +60,7 @@ module WebMock
           connection.once(:unmock_connection) do
             next unless connection.current_session == self
 
-            unless connection.addresses
+            unless connection.addresses?
               # reset Happy Eyeballs, fail early
               connection.sibling = nil
 

lib/httpx/connection.rb

@@ -122,6 +122,10 @@ module HTTPX
       @io && @io.addresses
     end
 
+    def addresses?
+      @io && @io.addresses?
+    end
+
     def match?(uri, options)
       return false if !used? && (@state == :closing || @state == :closed)
 
@@ -539,7 +543,7 @@ module HTTPX
 
     def send_request_to_parser(request)
       @inflight += 1
-      request.peer_address = @io.ip
+      request.peer_address = @io.ip && @io.ip.address
       set_request_timeouts(request)
 
       parser.send(request)

lib/httpx/io/tcp.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   class TCP
@@ -30,7 +29,8 @@ module HTTPX
         end
         raise Error, "Given IO objects do not match the request authority" unless @io
 
-        _, _, _, @ip = @io.addr
+        _, _, _, ip = @io.addr
+        @ip = Resolver::Entry.new(ip)
         @addresses << @ip
         @keep_open = true
         @state = :connected
@@ -47,8 +47,6 @@ module HTTPX
     def add_addresses(addrs)
       return if addrs.empty?
 
-      addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
-
       ip_index = @ip_index || (@addresses.size - 1)
       if addrs.first.ipv6?
         # should be the next in line
@@ -59,6 +57,13 @@ module HTTPX
       end
     end
 
+    # eliminates expired entries and returns whether there are still any left.
+    def addresses?
+      @addresses.delete_if(&:expired?)
+
+      @addresses.any?
+    end
+
     def to_io
       @io.to_io
     end
@@ -167,7 +172,7 @@ module HTTPX
       # do not mess with external sockets
       return false if @options.io
 
-      return true unless @addresses
+      return true if @addresses.empty?
 
       resolver_addresses = Resolver.nolookup_resolve(@hostname)
 

lib/httpx/io/unix.rb

@@ -51,6 +51,11 @@ module HTTPX
            IO::WaitReadable
     end
 
+    # the path is always explicitly passed, so no point in resolving.
+    def addresses?
+      true
+    end
+
     def expired?
       false
     end

lib/httpx/options.rb

@@ -210,7 +210,7 @@ module HTTPX
     end
 
     def option_addresses(value)
-      Array(value)
+      Array(value).map { |entry| Resolver::Entry.convert(entry) }
     end
 
     def option_ip_families(value)

lib/httpx/plugins/ssrf_filter.rb

@@ -129,8 +129,6 @@ module HTTPX
         end
 
         def addresses=(addrs)
-          addrs = addrs.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
-
           addrs.reject!(&SsrfFilter.method(:unsafe_ip_address?))
 
           raise ServerSideRequestForgeryError, "#{@origin.host} has no public IP addresses" if addrs.empty?

lib/httpx/request/body.rb

@@ -127,8 +127,13 @@ module HTTPX
           # @type var body: bodyIO
           Transcoder::Body.encode(body)
         elsif (form = params.delete(:form))
-          # @type var form: Transcoder::urlencoded_input
+          if Transcoder::Multipart.multipart?(form)
-          Transcoder::Form.encode(form)
+            # @type var form: Transcoder::multipart_input
+            Transcoder::Multipart.encode(form)
+          else
+            # @type var form: Transcoder::urlencoded_input
+            Transcoder::Form.encode(form)
+          end
         elsif (json = params.delete(:json))
           # @type var body: _ToJson
           Transcoder::JSON.encode(json)

lib/httpx/resolver.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   module Resolver
     RESOLVE_TIMEOUT = [2, 3].freeze
 
+    require "httpx/resolver/entry"
     require "httpx/resolver/resolver"
     require "httpx/resolver/system"
     require "httpx/resolver/native"
@@ -39,16 +39,19 @@ module HTTPX
       ip_resolve(hostname) || cached_lookup(hostname) || system_resolve(hostname)
     end
 
+    # tries to convert +hostname+ into an IPAddr, returns <tt>nil</tt> otherwise.
     def ip_resolve(hostname)
-      [IPAddr.new(hostname)]
+      [Entry.new(hostname)]
     rescue ArgumentError
     end
 
+    # matches +hostname+ to entries in the hosts file, returns <tt>nil</nil> if none is
+    # found, or there is no hosts file.
     def system_resolve(hostname)
       ips = @system_resolver.getaddresses(hostname)
       return if ips.empty?
 
-      ips.map { |ip| IPAddr.new(ip) }
+      ips.map { |ip| Entry.new(ip) }
     rescue IOError
     end
 
@@ -108,7 +111,7 @@ module HTTPX
         if (als = address["alias"])
           lookup(als, lookups, ttl)
         else
-          IPAddr.new(address["data"])
+          Entry.new(address["data"], address["TTL"])
         end
       end.compact
 

lib/httpx/resolver/entry.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "ipaddr"
+
+module HTTPX
+  module Resolver
+    class Entry < SimpleDelegator
+      attr_reader :address
+
+      def self.convert(address)
+        new(address, rescue_on_convert: true)
+      end
+
+      def initialize(address, expires_in = Float::INFINITY, rescue_on_convert: false)
+        @expires_in = expires_in
+        @address = address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
+        super(@address)
+      rescue IPAddr::InvalidAddressError
+        raise unless rescue_on_convert
+
+        @address = address.to_s
+        super(@address)
+      end
+
+      def expired?
+        @expires_in < Utils.now
+      end
+    end
+  end
+end

lib/httpx/resolver/native.rb

@@ -450,7 +450,7 @@ module HTTPX
       when :tcp
         log { "resolver #{FAMILY_TYPES[@record_type]}: server: tcp://#{ip}:#{port}..." }
         origin = URI("tcp://#{ip}:#{port}")
-        TCP.new(origin, [ip], @options)
+        TCP.new(origin, [Resolver::Entry.new(ip)], @options)
       end
     end
 

lib/httpx/resolver/resolver.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "ipaddr"
 
 module HTTPX
   # Base class for all internal internet name resolvers. It handles basic blocks
@@ -69,9 +68,7 @@ module HTTPX
     end
 
     def emit_addresses(connection, family, addresses, early_resolve = false)
-      addresses.map! do |address|
+      addresses.map! { |address| address.is_a?(Resolver::Entry) ? address : Resolver::Entry.new(address) }
-        address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
-      end
 
       # double emission check, but allow early resolution to work
       return if !early_resolve && connection.addresses && !addresses.intersect?(connection.addresses)

lib/httpx/session.rb

@@ -370,7 +370,7 @@ module HTTPX
     end
 
     def resolve_connection(connection, selector)
-      if connection.addresses || connection.open?
+      if connection.addresses? || connection.open?
         #
         # there are two cases in which we want to activate initialization of
         # connection immediately:

lib/httpx/transcoder/form.rb

@@ -48,11 +48,7 @@ module HTTPX
       end
 
       def encode(form)
-        if multipart?(form)
+        Encoder.new(form)
-          Multipart::Encoder.new(form)
-        else
-          Encoder.new(form)
-        end
       end
 
       def decode(response)
@@ -67,14 +63,6 @@ module HTTPX
           raise Error, "invalid form mime type (#{content_type})"
         end
       end
-
-      def multipart?(data)
-        data.any? do |_, v|
-          Multipart::MULTIPART_VALUE_COND.call(v) ||
-            (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
-            (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
-        end
-      end
     end
   end
 end

lib/httpx/transcoder/multipart.rb

@@ -13,5 +13,19 @@ module HTTPX::Transcoder
           value.key?(:body) &&
           (value.key?(:filename) || value.key?(:content_type)))
     end
+
+    module_function
+
+    def multipart?(form_data)
+      form_data.any? do |_, v|
+        Multipart::MULTIPART_VALUE_COND.call(v) ||
+          (v.respond_to?(:to_ary) && v.to_ary.any?(&Multipart::MULTIPART_VALUE_COND)) ||
+          (v.respond_to?(:to_hash) && v.to_hash.any? { |_, e| Multipart::MULTIPART_VALUE_COND.call(e) })
+      end
+    end
+
+    def encode(form_data)
+      Encoder.new(form_data)
+    end
   end
 end

sig/connection.rbs

@@ -50,14 +50,16 @@ module HTTPX
     @main_sibling: bool
 
 
-    def addresses: () -> Array[ipaddr]?
+    def addresses: () -> Array[Resolver::Entry]?
 
     def peer: () -> URI::Generic
 
-    def addresses=: (Array[ipaddr] addresses) -> void
+    def addresses=: (Array[Resolver::Entry] addresses) -> void
 
     def send: (Request request) -> void
 
+    def addresses?: () -> boolish
+
     def match?: (URI::Generic uri, Options options) -> bool
 
     def expired?: () -> boolish
@@ -140,7 +142,7 @@ module HTTPX
 
     def build_altsvc_connection: (URI::Generic alt_origin, String origin, Hash[String, String] alt_params) -> void
 
-    def build_socket: (?Array[ipaddr]? addrs) -> (TCP | SSL | UNIX)
+    def build_socket: (?Array[Resolver::Entry]? addrs) -> (TCP | SSL | UNIX)
 
     def on_error: (HTTPX::TimeoutError | Error | StandardError error, ?Request? request) -> void
 

sig/io/ssl.rbs

@@ -12,7 +12,7 @@ module HTTPX
     attr_writer ssl_session: OpenSSL::SSL::Session?
 
     # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
-    # def initialize: (URI::Generic origin, Array[ipaddr]? addresses, options options) ?{ (self) -> void } -> void
+    # def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, options options) ?{ (self) -> void } -> void
 
     def session_new_cb: { (OpenSSL::SSL::Session sess) -> void } -> void
     def can_verify_peer?: () -> bool

sig/io/tcp.rbs

@@ -2,11 +2,11 @@ module HTTPX
   class TCP
     include Loggable
 
-    attr_reader ip: ipaddr?
+    attr_reader ip: Resolver::Entry?
 
     attr_reader port: Integer
 
-    attr_reader addresses: Array[ipaddr]
+    attr_reader addresses: Array[Resolver::Entry]
 
     attr_reader state: Symbol
 
@@ -27,9 +27,11 @@ module HTTPX
     @ip_index: Integer
 
     # TODO: lift when https://github.com/ruby/rbs/issues/1497 fixed
-    def initialize: (URI::Generic origin, Array[ipaddr]? addresses, Options options) ?{ (instance) -> void } -> void
+    def initialize: (URI::Generic origin, Array[Resolver::Entry]? addresses, Options options) ?{ (instance) -> void } -> void
 
-    def add_addresses: (Array[ipaddr] addrs) -> void
+    def add_addresses: (Array[Resolver::Entry] addrs) -> void
+
+    def addresses?: () -> bool
 
     def to_io: () -> IO
 

sig/options.rbs

@@ -53,7 +53,7 @@ module HTTPX
     attr_reader transport: io_type | nil
 
     # addresses
-    attr_reader addresses: Array[ipaddr]?
+    attr_reader addresses: Array[Resolver::Entry]?
 
     # supported_compression_formats
     attr_reader supported_compression_formats: Array[String]

sig/plugins/ssrf_filter.rbs

@@ -8,7 +8,7 @@ module HTTPX
 
       IPV6_BLACKLIST: Array[[IPAddr, IPAddr]]
 
-      def self?.unsafe_ip_address?: (IPAddr) -> bool
+      def self?.unsafe_ip_address?: (Resolver::Entry ipaddr) -> bool
 
       interface _Options
         def allowed_schemes: () -> Array[String]

sig/request.rbs

@@ -17,7 +17,7 @@ module HTTPX
     attr_reader drain_error: StandardError?
     attr_reader active_timeouts: Array[Symbol]
 
-    attr_accessor peer_address: ipaddr?
+    attr_accessor peer_address: (String | IPAddr)?
 
     attr_writer persistent: bool
 

sig/resolver.rbs

@@ -1,5 +1,5 @@
 module HTTPX
-  type ipaddr = IPAddr | String
+  type ipaddr = String | IPAddr | Resolv::IPv4 | Resolver::Entry
 
   module Resolver
     type dns_resource = singleton(Resolv::DNS::Resource)
@@ -17,21 +17,21 @@ module HTTPX
 
     type dns_decoding_response = [:ok, Array[dns_result]] | [:decode_error, Resolv::DNS::DecodeError] | [:dns_error, Integer] | Symbol
 
-    def self?.nolookup_resolve: (String hostname) -> Array[IPAddr]?
+    def self?.nolookup_resolve: (String hostname) -> Array[Entry]?
 
-    def self?.ip_resolve: (String hostname) -> Array[IPAddr]?
+    def self?.ip_resolve: (String hostname) -> Array[Entry]?
 
-    def self?.system_resolve: (String hostname) -> Array[IPAddr]?
+    def self?.system_resolve: (String hostname) -> Array[Entry]?
 
     def self?.resolver_for: (Symbol | singleton(Resolver) resolver_type, Options options) -> singleton(Resolver)
 
-    def self?.cached_lookup: (String hostname) -> Array[IPAddr]?
+    def self?.cached_lookup: (String hostname) -> Array[Entry]?
 
     def self?.cached_lookup_set: (String hostname, ip_family family, Array[dns_result] addresses) -> void
 
-    def self?.cached_lookup_evict: (String hostname, ipaddr ip) -> void
+    def self?.cached_lookup_evict: (String hostname, _ToS ip) -> void
 
-    def self?.lookup: (String hostname, Hash[String, Array[dns_result]] lookups, Numeric ttl) -> Array[IPAddr]?
+    def self?.lookup: (String hostname, Hash[String, Array[dns_result]] lookups, Numeric ttl) -> Array[Entry]?
 
     def self?.generate_id: () -> Integer
 

sig/resolver/entry.rbs

@@ -0,0 +1,13 @@
+module HTTPX
+  class Resolver::Entry
+    attr_reader address: IPAddr | String
+
+    @expires_in: Numeric
+
+    def initialize: (IPAddr | _ToS address, ?Numeric expires_in, ?rescue_on_convert: boolish) -> void
+
+    def expired?: () -> bool
+
+    def self.convert: (IPAddr | _ToS address) -> Resolver::Entry
+  end
+end

sig/resolver/resolver.rbs

@@ -35,7 +35,7 @@ module HTTPX
 
       def each_connection: () { (Connection connection) -> void } -> void
 
-      def emit_addresses: (Connection connection, ip_family family, Array[IPAddr], ?bool early_resolve) -> void
+      def emit_addresses: (Connection connection, ip_family family, Array[ipaddr], ?bool early_resolve) -> void
 
       def self.multi?: () -> bool
 
@@ -43,7 +43,7 @@ module HTTPX
 
       def resolve: (?Connection connection, ?String hostname) -> void
 
-      def emit_resolved_connection: (Connection connection, Array[IPAddr] addresses, bool early_resolve) -> void
+      def emit_resolved_connection: (Connection connection, Array[ipaddr] addresses, bool early_resolve) -> void
 
       def initialize: (ip_family family, Options options) -> void
 

sig/response.rbs

@@ -35,7 +35,7 @@ module HTTPX
 
     def uri: () -> URI::Generic
 
-    def peer_address: () -> ipaddr?
+    def peer_address: () -> (String | IPAddr)?
 
     def merge_headers: (_Each[[String, headers_value]]) -> void
 
@@ -90,7 +90,7 @@ module HTTPX
 
     def uri: () -> URI::Generic
 
-    def peer_address: () -> ipaddr?
+    def peer_address: () -> (String | IPAddr)?
 
     def close: () -> void
 

sig/transcoder/form.rbs

@@ -3,14 +3,14 @@ module HTTPX::Transcoder
 
   type form_nested_value = form_value | _ToAry[form_value] | _ToHash[string, form_value]
 
-  type urlencoded_input = Enumerable[[_ToS, form_nested_value | Multipart::multipart_nested_value]]
+  type urlencoded_input = Enumerable[[_ToS, form_nested_value]]
 
   module Form
     PARAM_DEPTH_LIMIT: Integer
 
-    def self?.encode: (urlencoded_input form) -> (Encoder | Multipart::Encoder)
+    def self?.encode: (urlencoded_input form) -> Encoder
+
     def self?.decode: (HTTPX::Response response) -> _Decoder
-    def self?.multipart?: (form_nested_value | Multipart::multipart_nested_value data) -> bool
 
     class Encoder
       extend Forwardable

sig/transcoder/multipart.rbs

@@ -16,6 +16,12 @@ module HTTPX
 
       type multipart_nested_value = multipart_value | _ToAry[multipart_value] | _ToHash[string, multipart_value]
 
+      type multipart_input = Enumerable[[_ToS, Multipart::multipart_nested_value]]
+
+      def self?.encode: (multipart_input form_data) -> Multipart::Encoder
+
+      def self?.multipart?: (form_nested_value | multipart_nested_value form_data) -> bool
+
       class Encoder
         @boundary: String
         @part_index: Integer
@@ -36,9 +42,9 @@ module HTTPX
 
         private
 
-        def to_parts: (Enumerable[[Symbol | string, multipart_nested_value]] multipart_data) -> Array[_Reader]
+        def to_parts: (multipart_input multipart_data) -> Array[_Reader]
 
-        def initialize: (Enumerable[[Symbol | string, multipart_nested_value]] multipart_data) -> untyped
+        def initialize: (multipart_input multipart_data) -> untyped
 
         def header_part: (String key, String content_type, String? filename) -> StringIO