Merge branch 'github-issue-5' into 'master'

fix for loop on resolution and retry on new connection

See merge request honeyryderchuck/httpx!215

Gemfile

@@ -116,6 +116,7 @@ group :assorted do
       gem "pry-byebug", "~> 3.4.3"
     else
       gem "pry-byebug"
+      gem "debug" if RUBY_VERSION >= "3.1.0"
     end
   end
 end

examples/hackernews_pages.rb

@@ -1,25 +1,36 @@
 require "httpx"
 require "oga"
 
+http = HTTPX.plugin(:compression).plugin(:persistent).with(timeout: { operation_timeut: 5, connect_timeout: 5})
+
 PAGES = (ARGV.first || 10).to_i
 pages = PAGES.times.map do |page|
   "https://news.ycombinator.com/?p=#{page+1}"
 end
 
-links = []
-HTTPX.plugin(:compression).get(*pages).each_with_index.map do |response, i|
+links = Array.new(PAGES) { [] }
+Array(http.get(*pages)).each_with_index.map do |response, i|
   if response.is_a?(HTTPX::ErrorResponse)
     puts "error: #{response.error}"
     next
   end
   html = Oga.parse_html(response.to_s)
-  page_links = html.css('.itemlist a.storylink').map{|link| link.get('href') }
+  # binding.irb
+  page_links = html.css('.itemlist a.titlelink').map{|link| link.get('href') }
   puts "page(#{i+1}): #{page_links.size}"
   if page_links.size == 0
     puts "error(#{response.status}) on page #{i+1}"
   end
-  links << page_links
+  # page_links.each do |link|
+  #   puts "link: #{link}"
+  #   links[i] << http.get(link)
+  # end
+  links[i].concat(http.get(*page_links))
 end
 
-links = links.flatten
-puts "Pages: #{PAGES}\t Links: #{links.size}"
+links = links.each_with_index do |pages, i|
+  puts "Page: #{i+1}\t Links: #{pages.size}"
+  pages.each do |page|
+    puts "URL: #{page.uri} (#{page.status})"
+  end
+end

lib/httpx/extensions.rb

@@ -83,22 +83,41 @@ module HTTPX
   end
 
   module ArrayExtensions
-    refine Array do
+    module FilterMap
+      refine Array do
 
-      def filter_map
-        return to_enum(:filter_map) unless block_given?
+        def filter_map
+          return to_enum(:filter_map) unless block_given?
 
-        each_with_object([]) do |item, res|
-          processed = yield(item)
-          res << processed if processed
+          each_with_object([]) do |item, res|
+            processed = yield(item)
+            res << processed if processed
+          end
         end
       end unless Array.method_defined?(:filter_map)
+    end
 
-      def sum(accumulator = 0, &block)
-        values = block_given? ? map(&block) : self
-        values.inject(accumulator, :+)
+    module Sum
+      refine Array do
+        def sum(accumulator = 0, &block)
+          values = block_given? ? map(&block) : self
+          values.inject(accumulator, :+)
+        end
       end unless Array.method_defined?(:sum)
     end
+
+    module Intersect
+      refine Array do
+        def intersect?(arr)
+          if size < arr.size
+            smaller = self
+          else
+            smaller, arr = arr, self
+          end
+          (array & smaller).size > 0
+        end
+      end unless Array.method_defined?(:intersect?)
+    end
   end
 
   module IOExtensions

lib/httpx/plugins/compression.rb

@@ -72,7 +72,7 @@ module HTTPX
       end
 
       module ResponseBodyMethods
-        using ArrayExtensions
+        using ArrayExtensions::FilterMap
 
         attr_reader :encodings
 

lib/httpx/pool.rb

@@ -7,7 +7,7 @@ require "httpx/resolver"
 
 module HTTPX
   class Pool
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
     extend Forwardable
 
     def_delegator :@timers, :after

lib/httpx/resolver/multi.rb

@@ -6,7 +6,7 @@ require "resolv"
 module HTTPX
   class Resolver::Multi
     include Callbacks
-    using ArrayExtensions
+    using ArrayExtensions::FilterMap
 
     attr_reader :resolvers
 

lib/httpx/resolver/resolver.rb

@@ -8,6 +8,8 @@ module HTTPX
     include Callbacks
     include Loggable
 
+    using ArrayExtensions::Intersect
+
     RECORD_TYPES = {
       Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
       Socket::AF_INET => Resolv::DNS::Resource::IN::A,
@@ -48,6 +50,10 @@ module HTTPX
       addresses.map! do |address|
         address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
       end
+
+      # double emission check
+      return if connection.addresses && !addresses.intersect?(connection.addresses)
+
       log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
       if @pool && # if triggered by early resolve, pool may not be here yet
          !connection.io &&
@@ -56,8 +62,12 @@ module HTTPX
          addresses.first.to_s != connection.origin.host.to_s
         log { "resolver: A response, applying resolution delay..." }
         @pool.after(0.05) do
-          connection.addresses = addresses
-          emit(:resolve, connection)
+          # double emission check
+          unless connection.addresses && addresses.intersect?(connection.addresses)
+
+            connection.addresses = addresses
+            emit(:resolve, connection)
+          end
         end
       else
         connection.addresses = addresses

lib/httpx/response.rb

@@ -310,9 +310,12 @@ module HTTPX
 
   class ErrorResponse
     include Loggable
+    extend Forwardable
 
     attr_reader :request, :error
 
+    def_delegator :@request, :uri
+
     def initialize(request, error, options)
       @request = request
       @error = error

lib/httpx/transcoder/body.rb

@@ -9,7 +9,7 @@ module HTTPX::Transcoder
     module_function
 
     class Encoder
-      using HTTPX::ArrayExtensions
+      using HTTPX::ArrayExtensions::Sum
       extend Forwardable
 
       def_delegator :@raw, :to_s

sig/response.rbs

@@ -96,6 +96,7 @@ module HTTPX
   class ErrorResponse
     include _Response
     include Loggable
+    extend Forwardable
 
     @options: Options
     @error: Exception
@@ -104,6 +105,8 @@ module HTTPX
 
     def status: () -> (Integer | _ToS)
 
+    def uri: () -> URI::Generic
+
     private
 
     def initialize: (Request, Exception, options) -> untyped