diff --git a/Gemfile b/Gemfile index cc30dd6f..20896089 100644 --- a/Gemfile +++ b/Gemfile @@ -116,6 +116,7 @@ group :assorted do gem "pry-byebug", "~> 3.4.3" else gem "pry-byebug" + gem "debug" if RUBY_VERSION >= "3.1.0" end end end diff --git a/examples/hackernews_pages.rb b/examples/hackernews_pages.rb index 999f6b20..6f09b1a5 100644 --- a/examples/hackernews_pages.rb +++ b/examples/hackernews_pages.rb @@ -1,25 +1,36 @@ require "httpx" require "oga" +http = HTTPX.plugin(:compression).plugin(:persistent).with(timeout: { operation_timeut: 5, connect_timeout: 5}) + PAGES = (ARGV.first || 10).to_i pages = PAGES.times.map do |page| "https://news.ycombinator.com/?p=#{page+1}" end -links = [] -HTTPX.plugin(:compression).get(*pages).each_with_index.map do |response, i| +links = Array.new(PAGES) { [] } +Array(http.get(*pages)).each_with_index.map do |response, i| if response.is_a?(HTTPX::ErrorResponse) puts "error: #{response.error}" next end html = Oga.parse_html(response.to_s) - page_links = html.css('.itemlist a.storylink').map{|link| link.get('href') } + # binding.irb + page_links = html.css('.itemlist a.titlelink').map{|link| link.get('href') } puts "page(#{i+1}): #{page_links.size}" if page_links.size == 0 puts "error(#{response.status}) on page #{i+1}" end - links << page_links + # page_links.each do |link| + # puts "link: #{link}" + # links[i] << http.get(link) + # end + links[i].concat(http.get(*page_links)) end -links = links.flatten -puts "Pages: #{PAGES}\t Links: #{links.size}" +links = links.each_with_index do |pages, i| + puts "Page: #{i+1}\t Links: #{pages.size}" + pages.each do |page| + puts "URL: #{page.uri} (#{page.status})" + end +end diff --git a/lib/httpx/extensions.rb b/lib/httpx/extensions.rb index b5a4c0a8..4a02b923 100644 --- a/lib/httpx/extensions.rb +++ b/lib/httpx/extensions.rb @@ -83,22 +83,41 @@ module HTTPX end module ArrayExtensions - refine Array do + module FilterMap + refine Array do - def filter_map - return to_enum(:filter_map) unless block_given? + def filter_map + return to_enum(:filter_map) unless block_given? - each_with_object([]) do |item, res| - processed = yield(item) - res << processed if processed + each_with_object([]) do |item, res| + processed = yield(item) + res << processed if processed + end end end unless Array.method_defined?(:filter_map) + end - def sum(accumulator = 0, &block) - values = block_given? ? map(&block) : self - values.inject(accumulator, :+) + module Sum + refine Array do + def sum(accumulator = 0, &block) + values = block_given? ? map(&block) : self + values.inject(accumulator, :+) + end end unless Array.method_defined?(:sum) end + + module Intersect + refine Array do + def intersect?(arr) + if size < arr.size + smaller = self + else + smaller, arr = arr, self + end + (array & smaller).size > 0 + end + end unless Array.method_defined?(:intersect?) + end end module IOExtensions diff --git a/lib/httpx/plugins/compression.rb b/lib/httpx/plugins/compression.rb index 84690a22..a6e782e6 100644 --- a/lib/httpx/plugins/compression.rb +++ b/lib/httpx/plugins/compression.rb @@ -72,7 +72,7 @@ module HTTPX end module ResponseBodyMethods - using ArrayExtensions + using ArrayExtensions::FilterMap attr_reader :encodings diff --git a/lib/httpx/pool.rb b/lib/httpx/pool.rb index f9738027..0ef0eed0 100644 --- a/lib/httpx/pool.rb +++ b/lib/httpx/pool.rb @@ -7,7 +7,7 @@ require "httpx/resolver" module HTTPX class Pool - using ArrayExtensions + using ArrayExtensions::FilterMap extend Forwardable def_delegator :@timers, :after diff --git a/lib/httpx/resolver/multi.rb b/lib/httpx/resolver/multi.rb index 013056d7..294034e1 100644 --- a/lib/httpx/resolver/multi.rb +++ b/lib/httpx/resolver/multi.rb @@ -6,7 +6,7 @@ require "resolv" module HTTPX class Resolver::Multi include Callbacks - using ArrayExtensions + using ArrayExtensions::FilterMap attr_reader :resolvers diff --git a/lib/httpx/resolver/resolver.rb b/lib/httpx/resolver/resolver.rb index b54c53ee..8e36a2d4 100644 --- a/lib/httpx/resolver/resolver.rb +++ b/lib/httpx/resolver/resolver.rb @@ -8,6 +8,8 @@ module HTTPX include Callbacks include Loggable + using ArrayExtensions::Intersect + RECORD_TYPES = { Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA, Socket::AF_INET => Resolv::DNS::Resource::IN::A, @@ -48,6 +50,10 @@ module HTTPX addresses.map! do |address| address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s) end + + # double emission check + return if connection.addresses && !addresses.intersect?(connection.addresses) + log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" } if @pool && # if triggered by early resolve, pool may not be here yet !connection.io && @@ -56,8 +62,12 @@ module HTTPX addresses.first.to_s != connection.origin.host.to_s log { "resolver: A response, applying resolution delay..." } @pool.after(0.05) do - connection.addresses = addresses - emit(:resolve, connection) + # double emission check + unless connection.addresses && addresses.intersect?(connection.addresses) + + connection.addresses = addresses + emit(:resolve, connection) + end end else connection.addresses = addresses diff --git a/lib/httpx/response.rb b/lib/httpx/response.rb index c52effaa..1245fa9e 100644 --- a/lib/httpx/response.rb +++ b/lib/httpx/response.rb @@ -310,9 +310,12 @@ module HTTPX class ErrorResponse include Loggable + extend Forwardable attr_reader :request, :error + def_delegator :@request, :uri + def initialize(request, error, options) @request = request @error = error diff --git a/lib/httpx/transcoder/body.rb b/lib/httpx/transcoder/body.rb index 7eaca58a..d612f294 100644 --- a/lib/httpx/transcoder/body.rb +++ b/lib/httpx/transcoder/body.rb @@ -9,7 +9,7 @@ module HTTPX::Transcoder module_function class Encoder - using HTTPX::ArrayExtensions + using HTTPX::ArrayExtensions::Sum extend Forwardable def_delegator :@raw, :to_s diff --git a/sig/response.rbs b/sig/response.rbs index 712e0262..af2bbcbe 100644 --- a/sig/response.rbs +++ b/sig/response.rbs @@ -96,6 +96,7 @@ module HTTPX class ErrorResponse include _Response include Loggable + extend Forwardable @options: Options @error: Exception @@ -104,6 +105,8 @@ module HTTPX def status: () -> (Integer | _ToS) + def uri: () -> URI::Generic + private def initialize: (Request, Exception, options) -> untyped