sharpetronics/httpx
1
0
Fork 0
mirror of https://github.com/HoneyryderChuck/httpx.git synced 2025-08-10 00:01:27 -04:00
Code Issues Projects Releases Wiki Activity

updated rubocop, added a few changes...

Browse Source
This commit is contained in:
HoneyryderChuck 2018-12-28 02:44:43 +00:00
parent cfccff1d8d
commit 622188c0ab
45 changed files with 209 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Gemfile
View File

@ -7,7 +7,7 @@ gemspec
gem "hanna-nouveau", require: false
gem "rake", "~> 12.3"
gem "rubocop", "~> 0.57.0", require: false
gem "rubocop", "~> 0.61.1", require: false
gem "simplecov", require: false
platform :mri do

4
lib/httpx/altsvc.rb
View File

@ -18,6 +18,7 @@ module HTTPX
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@altsvc_mutex.synchronize do
return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
@altsvcs[origin] << entry
entry
@ -26,6 +27,7 @@ module HTTPX
def lookup(origin, ttl)
return [] unless @altsvcs.key?(origin)
@altsvcs[origin] = @altsvcs[origin].select do |entry|
!entry.key?("TTL") || entry["TTL"] > ttl
end
@ -35,6 +37,7 @@ module HTTPX
def emit(request, response)
# Alt-Svc
return unless response.headers.key?("alt-svc")
origin = request.origin
host = request.uri.host
parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
@ -45,6 +48,7 @@ module HTTPX
def parse(altsvc)
return enum_for(__method__, altsvc) unless block_given?
alt_origins, *alt_params = altsvc.split(/ *; */)
alt_params = Hash[alt_params.map { |field| field.split("=") }]
alt_origins.split(/ *, */).each do |alt_origin|

1
lib/httpx/callbacks.rb
View File

@ -25,6 +25,7 @@ module HTTPX
def callbacks(type = nil)
return @callbacks unless type
@callbacks ||= Hash.new { |h, k| h[k] = [] }
@callbacks[type]
end

1
lib/httpx/chainable.rb
View File

@ -49,6 +49,7 @@ module HTTPX
# :nodoc:
def branch(options, &blk)
return self.class.new(options, &blk) if is_a?(Client)
Client.new(options, &blk)
end
end

9
lib/httpx/channel.rb
View File

@ -86,6 +86,7 @@ module HTTPX
def mergeable?(addresses)
return false if @state == :closing || !@io
!(@io.addresses & addresses).empty?
end
@ -147,6 +148,7 @@ module HTTPX
def interests
return :w if @state == :idle
readable = !@read_buffer.full?
writable = !@write_buffer.empty?
if readable
@ -208,8 +210,10 @@ module HTTPX
def handle_timeout_error(e)
return emit(:error, e) unless @timeout
@timeout -= e.timeout
return unless @timeout <= 0
if connecting?
emit(:error, e.to_connection_error)
else
@ -237,6 +241,7 @@ module HTTPX
return
end
return if siz.zero?
log { "READ: #{siz} bytes..." }
parser << @read_buffer.to_s
return if @state == :closing || @state == :closed
@ -246,6 +251,7 @@ module HTTPX
def dwrite
loop do
return if @write_buffer.empty?
siz = @io.write(@write_buffer)
unless siz
ex = EOFError.new("descriptor closed")
@ -321,8 +327,10 @@ module HTTPX
@timeout = @timeout_threshold
when :open
return if @state == :closed
@io.connect
return unless @io.connected?
send_pending
@timeout_threshold = @options.timeout.operation_timeout
@timeout = @timeout_threshold
@ -332,6 +340,7 @@ module HTTPX
when :closed
return unless @state == :closing
return unless @write_buffer.empty?
@io.close
@read_buffer.clear
end

6
lib/httpx/channel/http1.rb
View File

@ -93,6 +93,7 @@ module HTTPX
def on_trailers(h)
return unless @request
response = @request.response
log(level: 2) { "trailer headers received" }
@ -102,6 +103,7 @@ module HTTPX
def on_data(chunk)
return unless @request
log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
log(level: 2, color: :green) { "-> #{chunk.inspect}" }
response = @request.response
@ -111,6 +113,7 @@ module HTTPX
def on_complete
return unless @request
log(level: 2) { "parsing complete" }
dispatch
end
@ -158,6 +161,7 @@ module HTTPX
when /keep\-alive/i
keep_alive = response.headers["keep-alive"]
return unless keep_alive
parameters = Hash[keep_alive.split(/ *, */).map do |pair|
pair.split(/ *= */)
end]
@ -176,6 +180,7 @@ module HTTPX
def disable_pipelining
return if @requests.empty?
@requests.each { |r| r.transition(:idle) }
# server doesn't handle pipelining, and probably
# doesn't support keep-alive. Fallback to send only
@ -226,6 +231,7 @@ module HTTPX
def join_body(request)
return if request.empty?
while (chunk = request.drain_body)
log(color: :green) { "<- DATA: #{chunk.bytesize} bytes..." }
log(level: 2, color: :green) { "<- #{chunk.inspect}" }

2
lib/httpx/channel/http2.rb
View File

@ -195,6 +195,7 @@ module HTTPX
def on_stream_close(stream, request, error)
return handle(request, stream) if request.expects?
if error
ex = Error.new(stream.id, error)
ex.set_backtrace(caller)
@ -233,6 +234,7 @@ module HTTPX
emit(:error, request, ex)
end
return unless @connection.state == :closed && @connection.active_stream_count.zero?
emit(:close)
end

2
lib/httpx/client.rb
View File

@ -15,6 +15,7 @@ module HTTPX
def wrap
return unless block_given?
begin
prev_keep_open = @keep_open
@keep_open = true
@ -125,6 +126,7 @@ module HTTPX
raise ArgumentError, "unsupported number of arguments"
end
raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
requests
end

3
lib/httpx/connection.rb
View File

@ -84,6 +84,7 @@ module HTTPX
@channels << channel unless @channels.include?(channel)
@resolver << channel
return if @resolver.empty?
@_resolver_monitor ||= begin # rubocop:disable Naming/MemoizedInstanceVariableName
monitor = @selector.register(@resolver, :w)
monitor.value = @resolver
@ -96,6 +97,7 @@ module HTTPX
ch != channel && ch.mergeable?(addresses)
end
return register_channel(channel) unless found_channel
if found_channel.state == :open
coalesce_channels(found_channel, channel)
else
@ -144,6 +146,7 @@ module HTTPX
def next_timeout
timeout = @timeout.timeout
return (@resolver.timeout || timeout) unless @resolver.closed?
timeout
end
end

4
lib/httpx/headers.rb
View File

@ -7,6 +7,7 @@ module HTTPX
class << self
def new(headers = nil)
return headers if headers.is_a?(self)
super
end
end
@ -14,6 +15,7 @@ module HTTPX
def initialize(headers = nil)
@headers = {}
return unless headers
headers.each do |field, value|
array_value(value).each do |v|
add(downcased(field), v)
@ -64,6 +66,7 @@ module HTTPX
#
def []=(field, value)
return unless value
@headers[downcased(field)] = array_value(value)
end
@ -94,6 +97,7 @@ module HTTPX
#
def each
return enum_for(__method__) { @headers.size } unless block_given?
@headers.each do |field, value|
yield(field, value.join(", ")) unless value.empty?
end

3
lib/httpx/io/ssl.rb
View File

@ -31,6 +31,7 @@ module HTTPX
def verify_hostname(host)
return false if @ctx.verify_mode == OpenSSL::SSL::VERIFY_NONE
return false if @io.peer_cert.nil?
OpenSSL::SSL.verify_certificate_identity(@io.peer_cert, host)
end
@ -54,6 +55,7 @@ module HTTPX
end
return if @state == :negotiated ||
@state != :connected
unless @io.is_a?(OpenSSL::SSL::SSLSocket)
@io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
@io.hostname = @hostname
@ -112,6 +114,7 @@ module HTTPX
def log_transition_state(nextstate)
return super unless nextstate == :negotiated
server_cert = @io.peer_cert
"SSL connection using #{@io.ssl_version} / #{@io.cipher.first}\n" \
"ALPN, server accepted to use #{protocol}\n" \

5
lib/httpx/io/tcp.rb
View File

@ -54,6 +54,7 @@ module HTTPX
def connect
return unless closed?
begin
if @io.closed?
transition(:idle)
@ -65,6 +66,7 @@ module HTTPX
transition(:connected)
rescue Errno::EHOSTUNREACH => e
raise e if @ip_index <= 0
@ip_index -= 1
retry
rescue Errno::EINPROGRESS,
@ -96,6 +98,7 @@ module HTTPX
ret = @io.read_nonblock(size, buffer, exception: false)
return 0 if ret == :wait_readable
return if ret.nil?
buffer.bytesize
end
@ -103,6 +106,7 @@ module HTTPX
siz = @io.write_nonblock(buffer, exception: false)
return 0 if siz == :wait_writable
return if siz.nil?
buffer.slice!(0, siz)
siz
end
@ -110,6 +114,7 @@ module HTTPX
def close
return if @keep_open || closed?
begin
@io.close
ensure

1
lib/httpx/io/udp.rb
View File

@ -48,6 +48,7 @@ module HTTPX
ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
return 0 if ret == :wait_readable
return if ret.nil?
buffer.bytesize
rescue IOError
end

1
lib/httpx/io/unix.rb
View File

@ -36,6 +36,7 @@ module HTTPX
def connect
return unless closed?
begin
if @io.closed?
transition(:idle)

15
lib/httpx/loggable.rb
View File

@ -3,19 +3,20 @@
module HTTPX
module Loggable
COLORS = {
black: 30,
red: 31,
green: 32,
yellow: 33,
blue: 34,
black: 30,
red: 31,
green: 32,
yellow: 33,
blue: 34,
magenta: 35,
cyan: 36,
white: 37,
cyan: 36,
white: 37,
}.freeze
def log(level: @options.debug_level, label: "", color: nil, &msg)
return unless @options.debug
return unless @options.debug_level >= level
message = (+label << msg.call << "\n")
message = "\e[#{COLORS[color]}m#{message}\e[0m" if color && @options.debug.isatty
@options.debug << message

39
lib/httpx/options.rb
View File

@ -16,6 +16,7 @@ module HTTPX
# let enhanced options go through
return options if self == Options && options.class > self
return options if options.is_a?(self)
super
end
@ -38,24 +39,24 @@ module HTTPX
def initialize(options = {})
defaults = {
:debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
:debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
:ssl => {},
:http2_settings => { settings_enable_push: 0 },
:fallback_protocol => "http/1.1",
:timeout => Timeout.new,
:headers => {},
:max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
:window_size => WINDOW_SIZE,
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
:request_class => Class.new(Request),
:response_class => Class.new(Response),
:headers_class => Class.new(Headers),
:request_body_class => Class.new(Request::Body),
:response_body_class => Class.new(Response::Body),
:transport => nil,
:transport_options => nil,
:resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
:debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
:debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
:ssl => {},
:http2_settings => { settings_enable_push: 0 },
:fallback_protocol => "http/1.1",
:timeout => Timeout.new,
:headers => {},
:max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
:window_size => WINDOW_SIZE,
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
:request_class => Class.new(Request),
:response_class => Class.new(Response),
:headers_class => Class.new(Headers),
:request_body_class => Class.new(Request::Body),
:response_body_class => Class.new(Response::Body),
:transport => nil,
:transport_options => nil,
:resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
}
defaults.merge!(options)
@ -74,6 +75,7 @@ module HTTPX
def_option(:max_concurrent_requests) do |num|
max = Integer(num)
raise Error, ":max_concurrent_requests must be positive" unless max.positive?
self.max_concurrent_requests = max
end
@ -88,6 +90,7 @@ module HTTPX
def_option(:transport) do |tr|
transport = tr.to_s
raise Error, "#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
self.transport = transport
end

8
lib/httpx/parser/http1.rb
View File

@ -57,13 +57,16 @@ module HTTPX
def parse_headline
idx = @buffer.index("\n")
return unless idx
(m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
raise(Error, "wrong head line format")
version, code, _ = m.captures
raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
@http_version = version.split(".").map(&:to_i)
@status_code = code.to_i
raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
@buffer.slice!(0, idx + 1)
nextstate(:headers)
end
@ -78,6 +81,7 @@ module HTTPX
prepare_data(headers)
@observer.on_headers(headers)
return unless @state == :headers
# state might have been reset
# in the :headers callback
nextstate(:data)
@ -93,12 +97,15 @@ module HTTPX
end
separator_index = line.index(@header_separator)
raise Error, "wrong header format" unless separator_index
key = line[0..separator_index - 1]
raise Error, "wrong header format" if key.start_with?("\s", "\t")
key.strip!
value = line[separator_index + 1..-1]
value.strip!
raise Error, "wrong header format" if value.nil?
(headers[key.downcase] ||= []) << value
end
end
@ -118,6 +125,7 @@ module HTTPX
@buffer.clear
end
return unless no_more_data?
@buffer = @buffer.to_s
if @_has_trailers
nextstate(:trailers)

3
lib/httpx/plugins/compression.rb
View File

@ -19,6 +19,7 @@ module HTTPX
def initialize(*)
super
return if @body.nil?
@headers.get("content-encoding").each do |encoding|
@body = Encoder.new(@body, Compression.registry(encoding).encoder)
end
@ -70,6 +71,7 @@ module HTTPX
def each(&blk)
return enum_for(__method__) unless block_given?
unless @buffer.size.zero?
@buffer.rewind
return @buffer.each(&blk)
@ -97,6 +99,7 @@ module HTTPX
def deflate(&blk)
return unless @buffer.size.zero?
@body.rewind
@deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
end

2
lib/httpx/plugins/cookies.rb
View File

@ -23,6 +23,7 @@ module HTTPX
module HeadersMethods
def cookies(jar, request)
return unless jar
unless jar.is_a?(HTTP::CookieJar)
jar = jar.each_with_object(HTTP::CookieJar.new) do |(k, v), j|
cookie = k.is_a?(HTTP::Cookie) ? v : HTTP::Cookie.new(k.to_s, v.to_s)
@ -39,6 +40,7 @@ module HTTPX
def cookie_jar
return @cookie_jar if defined?(@cookie_jar)
return nil unless headers.key?("set-cookie")
@cookie_jar ||= begin
jar = HTTP::CookieJar.new
jar.parse(headers["set-cookie"], @request.uri)

2
lib/httpx/plugins/digest_authentication.rb
View File

@ -19,6 +19,7 @@ module HTTPX
def request(*args, keep_open: @keep_open, **options)
return super unless @_digest
begin
requests = __build_reqs(*args, **options)
probe_request = requests.first
@ -39,6 +40,7 @@ module HTTPX
prev_response = response
end
return responses.first if responses.size == 1
responses
ensure
close unless keep_open

6
lib/httpx/plugins/follow_redirects.rb
View File

@ -6,7 +6,7 @@ module HTTPX
module FollowRedirects
module InstanceMethods
MAX_REDIRECTS = 3
REDIRECT_STATUS = 300..399
REDIRECT_STATUS = (300..399).freeze
def max_redirects(n)
branch(default_options.with_max_redirects(n.to_i))
@ -25,6 +25,7 @@ module HTTPX
redirect_requests = []
indexes = responses.each_with_index.map do |response, index|
next unless REDIRECT_STATUS.include?(response.status)
request = requests[index]
retry_request = __build_redirect_req(request, response, options)
redirect_requests << retry_request
@ -32,6 +33,7 @@ module HTTPX
end.compact
break if redirect_requests.empty?
break if max_redirects <= 0
max_redirects -= 1
redirect_responses = __send_reqs(*redirect_requests)
@ -42,6 +44,7 @@ module HTTPX
end
return responses.first if responses.size == 1
responses
ensure
@keep_open = keep_open
@ -88,6 +91,7 @@ module HTTPX
klass.def_option(:max_redirects) do |num|
num = Integer(num)
raise Error, ":max_redirects must be positive" unless num.positive?
num
end

3
lib/httpx/plugins/h2c.rb
View File

@ -10,11 +10,13 @@ module HTTPX
module InstanceMethods
def request(*args, keep_open: @keep_open, **options)
return super if @_h2c_probed
begin
requests = __build_reqs(*args, **options)
upgrade_request = requests.first
return super unless valid_h2c_upgrade_request?(upgrade_request)
upgrade_request.headers["upgrade"] = "h2c"
upgrade_request.headers.add("connection", "upgrade")
upgrade_request.headers.add("connection", "http2-settings")
@ -40,6 +42,7 @@ module HTTPX
responses = [upgrade_response] + __send_reqs(*requests[1..-1])
end
return responses.first if responses.size == 1
responses
ensure
@_h2c_probed = true

2
lib/httpx/plugins/proxy.rb
View File

@ -51,11 +51,13 @@ module HTTPX
uri = URI(request.uri)
proxy = proxy_params(uri)
raise Error, "Failed to connect to proxy" unless proxy
@connection.find_channel(proxy) || build_channel(proxy, options)
end
def build_channel(proxy, options)
return super if proxy.is_a?(URI::Generic)
channel = build_proxy_channel(proxy, **options)
set_channel_callbacks(channel, options)
channel

4
lib/httpx/plugins/proxy/http.rb
View File

@ -30,8 +30,10 @@ module HTTPX
case nextstate
when :connecting
return unless @state == :idle
@io.connect
return unless @io.connected?
@parser = ConnectProxyParser.new(@write_buffer, @options.merge(max_concurrent_requests: 1))
@parser.once(:response, &method(:on_connect))
@parser.on(:close) { transition(:closing) }
@ -39,6 +41,7 @@ module HTTPX
return if @state == :connected
when :connected
return unless @state == :idle || @state == :connecting
case @state
when :connecting
@parser.close
@ -85,6 +88,7 @@ module HTTPX
def headline_uri(request)
return super unless request.verb == :connect
uri = request.uri
tunnel = "#{uri.hostname}:#{uri.port}"
log { "establishing HTTP proxy tunnel to #{tunnel}" }

5
lib/httpx/plugins/proxy/socks4.rb
View File

@ -38,15 +38,19 @@ module HTTPX
case nextstate
when :connecting
return unless @state == :idle
@io.connect
return unless @io.connected?
req, _ = @pending.first
return unless req
request_uri = req.uri
@write_buffer << Packet.connect(@parameters, request_uri)
proxy_connect
when :connected
return unless @state == :connecting
@parser = nil
end
log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
@ -92,6 +96,7 @@ module HTTPX
begin
ip = IPAddr.new(uri.host)
raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
packet << [ip.to_i].pack("N")
rescue IPAddr::InvalidAddressError
if parameters.uri.scheme == "socks4"

7
lib/httpx/plugins/proxy/socks5.rb
View File

@ -53,6 +53,7 @@ module HTTPX
version, status = packet.unpack("CC")
check_version(version)
return transition(:negotiating) if status == SUCCESS
on_socks_error("socks authentication error: #{status}")
when :negotiating
version, reply, = packet.unpack("CC")
@ -70,20 +71,25 @@ module HTTPX
case nextstate
when :connecting
return unless @state == :idle
@io.connect
return unless @io.connected?
@write_buffer << Packet.negotiate(@parameters)
proxy_connect
when :authenticating
return unless @state == :connecting
@write_buffer << Packet.authenticate(@parameters)
when :negotiating
return unless @state == :connecting || @state == :authenticating
req, _ = @pending.first
request_uri = req.uri
@write_buffer << Packet.connect(request_uri)
when :connected
return unless @state == :negotiating
@parser = nil
end
log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
@ -147,6 +153,7 @@ module HTTPX
begin
ip = IPAddr.new(uri.host)
raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
packet << [IPV4, ip.to_i].pack("CN")
rescue IPAddr::InvalidAddressError
packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")

1
lib/httpx/plugins/push_promise.rb
View File

@ -53,6 +53,7 @@ module HTTPX
def __on_promise_response(parser, stream, h)
request = @promise_headers.delete(stream)
return unless request
parser.__send__(:on_stream_headers, stream, request, h)
request.transition(:done)
response = request.response

1
lib/httpx/plugins/retries.rb
View File

@ -42,6 +42,7 @@ module HTTPX
klass.def_option(:max_retries) do |num|
num = Integer(num)
raise Error, ":max_retries must be positive" unless num.positive?
num
end
end

2
lib/httpx/registry.rb
View File

@ -58,8 +58,10 @@ module HTTPX
def registry(tag = nil)
@registry ||= {}
return @registry if tag.nil?
handler = @registry.fetch(tag)
raise(Error, "#{tag} is not registered in #{self}") unless handler
case handler
when Symbol, String
const_get(handler)

7
lib/httpx/request.rb
View File

@ -84,6 +84,7 @@ module HTTPX
def query
return @query if defined?(@query)
query = []
if (q = @options.params)
query << URI.encode_www_form(q)
@ -94,6 +95,7 @@ module HTTPX
def drain_body
return nil if @body.nil?
@drainer ||= @body.each
chunk = @drainer.next
chunk.dup
@ -109,6 +111,7 @@ module HTTPX
class << self
def new(*, options)
return options.body if options.body.is_a?(self)
super
end
end
@ -123,6 +126,7 @@ module HTTPX
Transcoder.registry("json").encode(options.json)
end
return if @body.nil?
@headers["content-type"] ||= @body.content_type
@headers["content-length"] = @body.bytesize unless unbounded_body?
end
@ -130,6 +134,7 @@ module HTTPX
def each(&block)
return enum_for(__method__) unless block_given?
return if @body.nil?
body = stream(@body)
if body.respond_to?(:read)
::IO.copy_stream(body, ProcIO.new(block))
@ -143,11 +148,13 @@ module HTTPX
def empty?
return true if @body.nil?
return false if chunked?
bytesize.zero?
end
def bytesize
return 0 if @body.nil?
if @body.respond_to?(:bytesize)
@body.bytesize
elsif @body.respond_to?(:size)

3
lib/httpx/resolver.rb
View File

@ -52,6 +52,7 @@ module HTTPX
# do not use directly!
def lookup(hostname, ttl)
return unless @lookups.key?(hostname)
@lookups[hostname] = @lookups[hostname].select do |address|
address["TTL"] > ttl
end
@ -92,7 +93,7 @@ module HTTPX
Resolv::DNS::Resource::IN::AAAA
addresses << {
"name" => question.to_s,
"TTL" => value.ttl,
"TTL" => value.ttl,
"data" => value.address.to_s,
}
end

17
lib/httpx/resolver/https.rb
View File

@ -56,6 +56,7 @@ module HTTPX
def closed?
return true unless @resolver_channel
resolver_channel.closed?
end
@ -67,6 +68,7 @@ module HTTPX
def resolve(channel = @channels.first, hostname = nil)
return if @building_channel
hostname = hostname || @queries.key(channel) || channel.uri.host
type = @_record_types[hostname].first
log(label: "resolver: ") { "query #{type} for #{hostname}" }
@ -114,12 +116,12 @@ module HTTPX
def parse(response)
answers = begin
decode_response_body(response)
rescue Resolv::DNS::DecodeError, JSON::JSONError => e
host, channel = @queries.first
if @_record_types[host].empty?
emit_resolve_error(channel, host, e)
return
end
rescue Resolv::DNS::DecodeError, JSON::JSONError => e
host, channel = @queries.first
if @_record_types[host].empty?
emit_resolve_error(channel, host, e)
return
end
end
if answers.empty?
host, channel = @queries.first
@ -148,15 +150,18 @@ module HTTPX
end
end.compact
next if addresses.empty?
hostname = hostname[0..-2] if hostname.end_with?(".")
channel = @queries.delete(hostname)
next unless channel # probably a retried query for which there's an answer
@channels.delete(channel)
Resolver.cached_lookup_set(hostname, addresses)
emit_addresses(channel, addresses.map { |addr| addr["data"] })
end
end
return if @channels.empty?
resolve
end

22
lib/httpx/resolver/native.rb
View File

@ -99,6 +99,7 @@ module HTTPX
def <<(channel)
return if early_resolve(channel)
if @nameserver.nil?
ex = ResolveError.new("Can't resolve #{channel.uri.host}: no nameserver")
ex.set_backtrace(caller)
@ -124,6 +125,7 @@ module HTTPX
def do_retry
return if @queries.empty?
loop_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_timeout
channels = []
queries = {}
@ -159,6 +161,7 @@ module HTTPX
return
end
return if siz.zero?
log(label: "resolver: ") { "READ: #{siz} bytes..." }
parse(@read_buffer.to_s)
end
@ -167,6 +170,7 @@ module HTTPX
def dwrite
loop do
return if @write_buffer.empty?
siz = @io.write(@write_buffer)
unless siz
emit(:close)
@ -180,12 +184,12 @@ module HTTPX
def parse(buffer)
addresses = begin
Resolver.decode_dns_answer(buffer)
rescue Resolv::DNS::DecodeError => e
hostname, channel = @queries.first
if @_record_types[hostname].empty?
emit_resolve_error(channel, hostname, e)
return
end
rescue Resolv::DNS::DecodeError => e
hostname, channel = @queries.first
if @_record_types[hostname].empty?
emit_resolve_error(channel, hostname, e)
return
end
end
if addresses.empty?
@ -200,6 +204,7 @@ module HTTPX
address = addresses.first
channel = @queries.delete(address["name"])
return unless channel # probably a retried query for which there's an answer
if address.key?("alias") # CNAME
if early_resolve(channel, hostname: address["alias"])
@channels.delete(channel)
@ -215,12 +220,14 @@ module HTTPX
end
end
return emit(:close) if @channels.empty?
resolve
end
def resolve(channel = @channels.first, hostname = nil)
raise Error, "no URI to resolve" unless channel
return unless @write_buffer.empty?
hostname = hostname || @queries.key(channel) || channel.uri.host
@queries[hostname] = channel
type = @_record_types[hostname].first
@ -234,6 +241,7 @@ module HTTPX
def build_socket
return if @io
ip, port = @nameserver[@ns_index]
port ||= DNS_PORT
uri = URI::Generic.build(scheme: "udp", port: port)
@ -253,11 +261,13 @@ module HTTPX
@timeouts.clear
when :open
return unless @state == :idle
build_socket
@io.connect
return unless @io.connected?
when :closed
return unless @state == :open
@io.close if @io
end
@state = nextstate

2
lib/httpx/resolver/resolver_mixin.rb
View File

@ -38,6 +38,7 @@ module HTTPX
def early_resolve(channel, hostname: channel.uri.host)
addresses = ip_resolve(hostname) || Resolver.cached_lookup(hostname) || system_resolve(hostname)
return unless addresses
emit_addresses(channel, addresses)
end
@ -49,6 +50,7 @@ module HTTPX
@system_resolver ||= Resolv::Hosts.new
ips = @system_resolver.getaddresses(hostname)
return if ips.empty?
ips.map { |ip| IPAddr.new(ip) }
end

1
lib/httpx/resolver/system.rb
View File

@ -32,6 +32,7 @@ module HTTPX
hostname = channel.uri.host
addresses = ip_resolve(hostname) || system_resolve(hostname) || @resolver.getaddresses(hostname)
return emit_resolve_error(channel, hostname) if addresses.empty?
emit_addresses(channel, addresses)
rescue Errno::EHOSTUNREACH, *RESOLV_ERRORS => e
emit_resolve_error(channel, hostname, e)

11
lib/httpx/response.rb
View File

@ -58,6 +58,7 @@ module HTTPX
def raise_for_status
return if @status < 400
raise HTTPError, self
end
@ -70,6 +71,7 @@ module HTTPX
@status == 304 || begin
content_length = @headers["content-length"]
return false if content_length.nil?
content_length == "0"
end
end
@ -94,6 +96,7 @@ module HTTPX
def read(*args)
return unless @buffer
@buffer.read(*args)
end
@ -103,6 +106,7 @@ module HTTPX
def each
return enum_for(__method__) unless block_given?
begin
unless @state == :idle
rewind
@ -137,6 +141,7 @@ module HTTPX
def copy_to(dest)
return unless @buffer
if dest.respond_to?(:path) && @buffer.respond_to?(:path)
FileUtils.mv(@buffer.path, dest.path)
else
@ -148,6 +153,7 @@ module HTTPX
# closes/cleans the buffer, resets everything
def close
return if @state == :idle
@buffer.close
@buffer.unlink if @buffer.respond_to?(:unlink)
@buffer = nil
@ -163,6 +169,7 @@ module HTTPX
def rewind
return if @state == :idle
@buffer.rewind
end
@ -197,8 +204,8 @@ module HTTPX
end
class ContentType
MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}
CHARSET_RE = /;\s*charset=([^;]+)/i
MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
attr_reader :mime_type, :charset

4
lib/httpx/selector.rb
View File

@ -31,6 +31,7 @@ class HTTPX::Selector
# closes +@io+, deregisters from reactor (unless +deregister+ is false)
def close(deregister = true)
return if @closed
@closed = true
@reactor.deregister(@io) if deregister
end
@ -125,6 +126,7 @@ class HTTPX::Selector
else
monitor = io.closed? ? @readers.delete(io) : @readers[io]
next unless monitor
monitor.readiness = writers.delete(io) ? :rw : :r
yield monitor
end
@ -133,6 +135,7 @@ class HTTPX::Selector
writers.each do |io|
monitor = io.closed? ? @writers.delete(io) : @writers[io]
next unless monitor
# don't double run this, the last iteration might have run this task already
monitor.readiness = :w
yield monitor
@ -143,6 +146,7 @@ class HTTPX::Selector
#
def close
return if @closed
@__r__.close
@__w__.close
rescue IOError

3
lib/httpx/timeout.rb
View File

@ -9,6 +9,7 @@ module HTTPX
def self.new(opts = {})
return opts if opts.is_a?(Timeout)
super
end
@ -65,6 +66,7 @@ module HTTPX
def transition(nextstate)
return if @state == nextstate
case nextstate
# when :idle
when :idle
@ -88,6 +90,7 @@ module HTTPX
def log_time
return unless @time_left
return reset_timer unless @started
@time_left -= (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started)
raise TimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds") if @time_left <= 0

4
lib/httpx/transcoder/chunker.rb
View File

@ -18,6 +18,7 @@ module HTTPX::Transcoder
def each
return enum_for(__method__) unless block_given?
@raw.each do |chunk|
yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
end
@ -57,6 +58,7 @@ module HTTPX::Transcoder
when :length
index = @buffer.index(CRLF)
return unless index && index.positive?
# Read hex-length
hexlen = @buffer.slice!(0, index)
hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
@ -69,11 +71,13 @@ module HTTPX::Transcoder
# consume CRLF
return if @buffer.bytesize < crlf_size
raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
@buffer.slice!(0, crlf_size)
if @chunk_length.nil?
nextstate(:length)
else
return if @finished
nextstate(:data)
end
when :data

46
test/options_test.rb
View File

@ -43,8 +43,8 @@ class OptionsTest < Minitest::Test
assert opts.merge(opt2).body == "short", "options parameter hasn't been merged"
foo = Options.new(
:form => { :foo => "foo" },
:headers => { :accept => "json", :foo => "foo" },
:form => { :foo => "foo" },
:headers => { :accept => "json", :foo => "foo" },
)
bar = Options.new(
@ -54,31 +54,31 @@ class OptionsTest < Minitest::Test
)
assert foo.merge(bar).to_hash == {
:io => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
:debug => nil,
:debug_level => 1,
:params => nil,
:json => nil,
:body => nil,
:follow => nil,
:window_size => 16_384,
:io => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
:debug => nil,
:debug_level => 1,
:params => nil,
:json => nil,
:body => nil,
:follow => nil,
:window_size => 16_384,
:body_threshold_size => 114_688,
:form => { :bar => "bar" },
:timeout => Timeout.new,
:ssl => { :foo => "bar" },
:http2_settings => { :settings_enable_push => 0 },
:fallback_protocol => "http/1.1",
:headers => { "Foo" => "foo", "Accept" => "xml", "Bar" => "bar" },
:form => { :bar => "bar" },
:timeout => Timeout.new,
:ssl => { :foo => "bar" },
:http2_settings => { :settings_enable_push => 0 },
:fallback_protocol => "http/1.1",
:headers => { "Foo" => "foo", "Accept" => "xml", "Bar" => "bar" },
:max_concurrent_requests => 100,
:request_class => bar.request_class,
:response_class => bar.response_class,
:headers_class => bar.headers_class,
:request_class => bar.request_class,
:response_class => bar.response_class,
:headers_class => bar.headers_class,
:request_body_class => bar.request_body_class,
:response_body_class => bar.response_body_class,
:transport => nil,
:transport_options => nil,
:resolver_class => bar.resolver_class,
:resolver_options => bar.resolver_options,
:transport => nil,
:transport_options => nil,
:resolver_class => bar.resolver_class,
:resolver_options => bar.resolver_options,
}, "options haven't merged correctly"
end

1
test/support/assertion_helpers.rb
View File

@ -5,6 +5,7 @@ module ResponseHelpers
def verify_status(response, expect)
raise response.status if response.is_a?(HTTPX::ErrorResponse)
assert response.status == expect, "status assertion failed: #{response.status} (expected: #{expect})"
end

2
test/support/proxy_helper.rb
View File

@ -63,8 +63,10 @@ module ProxyHelper
def proxies_list(document)
row = document.enum_for(:each_node).find do |node|
next unless node.is_a?(Oga::XML::Element)
id = node.attribute("id")
next unless id
id.value == "proxylisttable"
end
row ? row.css("tr") : []

2
test/support/proxy_retry.rb
View File

@ -1,10 +1,12 @@
module ProxyRetry
def run(*)
return super unless name.include?("_proxy")
result = nil
3.times.each do |_i|
result = super
break if result.passed?
self.failures = []
self.assertions = 0
end

1
test/support/requests/response_body.rb
View File

@ -47,6 +47,7 @@ module Requests
def close
return unless @file
@file.close
@file.unlink
end

4
test/support/resolver_helpers.rb
View File

@ -38,6 +38,7 @@ module ResolverHelpers
def test_parse_a_record
return unless resolver.respond_to?(:parse)
channel = build_channel("http://ipv4.tlund.se/")
resolver.queries["ipv4.tlund.se"] = channel
resolver.parse(a_record)
@ -46,6 +47,7 @@ module ResolverHelpers
def test_parse_aaaa_record
return unless resolver.respond_to?(:parse)
channel = build_channel("http://ipv6.tlund.se/")
resolver.queries["ipv6.tlund.se"] = channel
resolver.parse(aaaa_record)
@ -54,6 +56,7 @@ module ResolverHelpers
def test_parse_cname_record
return unless resolver.respond_to?(:parse)
channel = build_channel("http://ipv4c.tlund.se/")
resolver.queries["ipv4c.tlund.se"] = channel
# require "pry-byebug"; binding.pry
@ -65,6 +68,7 @@ module ResolverHelpers
def test_append_hostname
return unless resolver.respond_to?(:resolve)
channel = build_channel("https://news.ycombinator.com")
resolver << channel
assert channel.addresses.nil?, "there should be no direct IP"