sharpetronics/httpx
1
0
Fork 0
mirror of https://github.com/HoneyryderChuck/httpx.git synced 2025-10-04 00:00:37 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'urlescaping' into 'master'

Browse Source 
Support for IDN domain names

See merge request honeyryderchuck/httpx!101
This commit is contained in:
HoneyryderChuck 2020-11-07 16:04:53 +00:00
commit bb1f448bb2
43 changed files with 430 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.rubocop-2.4.yml
View File

@ -1 +1,7 @@
inherit_from: .rubocop-2.3.yml
Style/OptionalBooleanParameter:
Enabled: false
Gemspec/RequiredRubyVersion:
Enabled: false

2
.rubocop-2.5.yml
View File

@ -1 +1 @@
inherit_from: .rubocop-2.3.yml
inherit_from: .rubocop-2.4.yml

2
.rubocop-2.6.yml
View File

@ -1 +1 @@
inherit_from: .rubocop-2.3.yml
inherit_from: .rubocop-2.4.yml

2
.rubocop-2.7.yml
View File

@ -1 +1 @@
inherit_from: .rubocop-2.3.yml
inherit_from: .rubocop-2.4.yml

2
.rubocop-3.0.yml
View File

@ -1 +1 @@
inherit_from: .rubocop-2.3.yml
inherit_from: .rubocop-2.4.yml

2
.rubocop.yml
View File

@ -1,7 +1,7 @@
inherit_from: .rubocop_todo.yml
AllCops:
TargetRubyVersion: 2.3
TargetRubyVersion: 2.4
DisplayCopNames: true
Include:
- lib/**/*.rb

2
.rubocop_todo.yml
View File

@ -36,4 +36,4 @@ Style/Documentation:
Naming/AccessorMethodName:
Enabled: false

6
Gemfile
View File

@ -16,8 +16,10 @@ group :test do
gem "net-ssh", "~> 4.2.0"
elsif RUBY_VERSION < "2.3"
gem "rubocop", "~> 0.68.1"
elsif RUBY_VERSION < "2.3"
gem "rubocop", "~> 0.81.1"
else
gem "rubocop", "~> 0.80.0"
gem "rubocop", "~> 1.0.0"
gem "rubocop-performance", "~> 1.5.2"
end
@ -57,7 +59,7 @@ group :website do
gem "jekyll", "~> 4.0.0"
gem "jekyll-gzip", "~> 2.4.1"
gem "jekyll-paginate-v2", "~> 1.5.2"
gem "jekyll-brotli", "~> 2.2.0"
gem "jekyll-brotli", "~> 2.2.0", platform: :mri
end if RUBY_VERSION > "2.4"
group :assorted do

2
LICENSE.txt
View File

@ -191,7 +191,7 @@
limitations under the License.
* lib/httpx/plugins/cookies/domain_name.rb
* lib/httpx/domain_name.rb
This file is derived from the implementation of punycode available at
here:

1
lib/httpx.rb
View File

@ -6,6 +6,7 @@ require "httpx/extensions"
require "httpx/errors"
require "httpx/utils"
require "httpx/domain_name"
require "httpx/altsvc"
require "httpx/callbacks"
require "httpx/loggable"

2
lib/httpx/adapters/faraday.rb
View File

@ -121,7 +121,7 @@ module Faraday
end
def respond_to_missing?(meth)
@env.respond_to?(meth)
@env.respond_to?(meth) || super
end
def method_missing(meth, *args, &blk)

9
lib/httpx/connection.rb
View File

@ -51,7 +51,7 @@ module HTTPX
def initialize(type, uri, options)
@type = type
@origins = [uri.origin]
@origin = URI(uri.origin)
@origin = Utils.uri(uri.origin)
@options = Options.new(options)
@window_size = @options.window_size
@read_buffer = Buffer.new(BUFFER_SIZE)
@ -142,7 +142,7 @@ module HTTPX
end
end
def purge_pending
def purge_pending(&block)
pendings = []
if @parser
@inflight -= @parser.pending.size
@ -150,9 +150,7 @@ module HTTPX
end
pendings << @pending
pendings.each do |pending|
pending.reject! do |request|
yield request
end
pending.reject!(&block)
end
end
@ -460,7 +458,6 @@ module HTTPX
throw(:jump_tick)
rescue Errno::ECONNREFUSED,
Errno::EADDRNOTAVAIL,
Errno::EHOSTUNREACH,
OpenSSL::SSL::SSLError => e
# connect errors, exit gracefully
handle_error(e)

2
lib/httpx/connection/http1.rb
View File

@ -181,7 +181,7 @@ module HTTPX
def manage_connection(response)
connection = response.headers["connection"]
case connection
when /keep\-alive/i
when /keep-alive/i
keep_alive = response.headers["keep-alive"]
return unless keep_alive

6
lib/httpx/plugins/cookies/domain_name.rb → lib/httpx/domain_name.rb
View File

@ -28,7 +28,7 @@
require "ipaddr"
module HTTPX::Plugins::Cookies
module HTTPX
# Represents a domain name ready for extracting its registered domain
# and TLD.
class DomainName
@ -63,7 +63,7 @@ module HTTPX::Plugins::Cookies
# Normalizes a _domain_ using the Punycode algorithm as necessary.
# The result will be a downcased, ASCII-only string.
def normalize(domain)
domain = domain.ascii_only? ? domain : domain.chomp(DOT).unicode_normaliza(:nfc)
domain = domain.ascii_only? ? domain : domain.chomp(DOT).unicode_normalize(:nfc)
Punycode.encode_hostname(domain).downcase
end
end
@ -249,7 +249,7 @@ module HTTPX::Plugins::Cookies
# Encode a +string+ in Punycode
def encode(string)
input = string.unpack("U*")
output = ""
output = +""
# Initialize the state
n = INITIAL_N

1
lib/httpx/errors.rb
View File

@ -28,6 +28,7 @@ module HTTPX
NativeResolveError = Class.new(ResolveError) do
attr_reader :connection, :host
def initialize(connection, host, message = "Can't resolve #{host}")
@connection = connection
@host = host

22
lib/httpx/extensions.rb
View File

@ -54,11 +54,31 @@ module HTTPX
Numeric.__send__(:include, NegMethods)
end
module RegexpExtensions
# If you wonder why this is there: the oauth feature uses a refinement to enhance the
# Regexp class locally with #match? , but this is never tested, because ActiveSupport
# monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
# :nocov:
refine(Regexp) do
def match?(*args)
!match(*args).nil?
end
end
end
module URIExtensions
refine URI::Generic do
def non_ascii_hostname
@non_ascii_hostname
end
def non_ascii_hostname=(hostname)
@non_ascii_hostname = hostname
end
def authority
port_string = port == default_port ? nil : ":#{port}"
"#{host}#{port_string}"
"#{@non_ascii_hostname || host}#{port_string}"
end
def origin

6
lib/httpx/io/tcp.rb
View File

@ -7,11 +7,7 @@ module HTTPX
class TCP
include Loggable
attr_reader :ip, :port
attr_reader :addresses
attr_reader :state
attr_reader :ip, :port, :addresses, :state
alias_method :host, :ip

2
lib/httpx/io/unix.rb
View File

@ -8,6 +8,7 @@ module HTTPX
def_delegator :@uri, :port, :scheme
# rubocop:disable Lint/MissingSuper
def initialize(uri, addresses, options)
@uri = uri
@addresses = addresses
@ -29,6 +30,7 @@ module HTTPX
end
@io ||= build_socket
end
# rubocop:enable Lint/MissingSuper
def hostname
@uri.host

2
lib/httpx/parser/http1.rb
View File

@ -57,7 +57,7 @@ module HTTPX
idx = @buffer.index("\n")
return unless idx
(m = %r{\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
(m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
raise(Error, "wrong head line format")
version, code, _ = m.captures
raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)

7
lib/httpx/plugins/compression.rb
View File

@ -46,11 +46,8 @@ module HTTPX
super
return if @body.nil?
if (threshold = options.compression_threshold_size)
unless unbounded_body?
return if @body.bytesize < threshold
end
end
threshold = options.compression_threshold_size
return if threshold && !unbounded_body? && @body.bytesize < threshold
@headers.get("content-encoding").each do |encoding|
next if encoding == "identity"

1
lib/httpx/plugins/cookies.rb
View File

@ -15,7 +15,6 @@ module HTTPX
def self.load_dependencies(*)
require "httpx/plugins/cookies/jar"
require "httpx/plugins/cookies/cookie"
require "httpx/plugins/cookies/domain_name"
require "httpx/plugins/cookies/set_cookie_parser"
end

310
lib/httpx/plugins/cookies/cookie.rb
View File

@ -1,176 +1,172 @@
# frozen_string_literal: true
module HTTPX::Plugins::Cookies
# The HTTP Cookie.
#
# Contains the single cookie info: name, value and attributes.
class Cookie
include Comparable
# Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
# least)
MAX_LENGTH = 4096
module HTTPX
module Plugins::Cookies
# The HTTP Cookie.
#
# Contains the single cookie info: name, value and attributes.
class Cookie
include Comparable
# Maximum number of bytes per cookie (RFC 6265 6.1 requires 4096 at
# least)
MAX_LENGTH = 4096
attr_reader :domain
attr_reader :domain, :path, :name, :value, :created_at
attr_reader :path
attr_reader :name, :value
attr_reader :created_at
def path=(path)
path = String(path)
@path = path.start_with?("/") ? path : "/"
end
# See #domain.
def domain=(domain)
domain = String(domain)
if domain.start_with?(".")
@for_domain = true
domain = domain[1..-1]
def path=(path)
path = String(path)
@path = path.start_with?("/") ? path : "/"
end
return if domain.empty?
# See #domain.
def domain=(domain)
domain = String(domain)
@domain_name = DomainName.new(domain)
# RFC 6265 5.3 5.
@for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
@domain = @domain_name.hostname
end
# Compares the cookie with another. When there are many cookies with
# the same name for a URL, the value of the smallest must be used.
def <=>(other)
# RFC 6265 5.4
# Precedence: 1. longer path 2. older creation
(@name <=> other.name).nonzero? ||
(other.path.length <=> @path.length).nonzero? ||
(@created_at <=> other.created_at).nonzero? ||
@value <=> other.value
end
class << self
def new(cookie, *args)
return cookie if cookie.is_a?(self)
super
end
# Tests if +target_path+ is under +base_path+ as described in RFC
# 6265 5.1.4. +base_path+ must be an absolute path.
# +target_path+ may be empty, in which case it is treated as the
# root path.
#
# e.g.
#
# path_match?('/admin/', '/admin/index') == true
# path_match?('/admin/', '/Admin/index') == false
# path_match?('/admin/', '/admin/') == true
# path_match?('/admin/', '/admin') == false
#
# path_match?('/admin', '/admin') == true
# path_match?('/admin', '/Admin') == false
# path_match?('/admin', '/admins') == false
# path_match?('/admin', '/admin/') == true
# path_match?('/admin', '/admin/index') == true
def path_match?(base_path, target_path)
base_path.start_with?("/") || (return false)
# RFC 6265 5.1.4
bsize = base_path.size
tsize = target_path.size
return bsize == 1 if tsize.zero? # treat empty target_path as "/"
return false unless target_path.start_with?(base_path)
return true if bsize == tsize || base_path.end_with?("/")
target_path[bsize] == "/"
end
end
def initialize(arg, *attrs)
@created_at = Time.now
if attrs.empty?
attr_hash = Hash.try_convert(arg)
else
@name = arg
@value, attr_hash = attrs
attr_hash = Hash.try_convert(attr_hash)
end
attr_hash.each do |key, val|
key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
case key
when :domain, :path
__send__(:"#{key}=", val)
else
instance_variable_set(:"@#{key}", val)
if domain.start_with?(".")
@for_domain = true
domain = domain[1..-1]
end
end if attr_hash
@path ||= "/"
raise ArgumentError, "name must be specified" if @name.nil?
end
return if domain.empty?
def expires
@expires || (@created_at && @max_age ? @created_at + @max_age : nil)
end
@domain_name = DomainName.new(domain)
# RFC 6265 5.3 5.
@for_domain = false if @domain_name.domain.nil? # a public suffix or IP address
def expired?(time = Time.now)
return false unless expires
expires <= time
end
# Returns a string for use in the Cookie header, i.e. `name=value`
# or `name="value"`.
def cookie_value
"#{@name}=#{Scanner.quote(@value)}"
end
alias_method :to_s, :cookie_value
# Tests if it is OK to send this cookie to a given `uri`. A
# RuntimeError is raised if the cookie's domain is unknown.
def valid_for_uri?(uri)
uri = URI(uri)
# RFC 6265 5.4
return false if @secure && uri.scheme != "https"
acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
end
private
# Tests if it is OK to accept this cookie if it is sent from a given
# URI/URL, `uri`.
def acceptable_from_uri?(uri)
uri = URI(uri)
host = DomainName.new(uri.host)
# RFC 6265 5.3
if host.hostname == @domain
true
elsif @for_domain # !host-only-flag
host.cookie_domain?(@domain_name)
else
@domain.nil?
@domain = @domain_name.hostname
end
end
module Scanner
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
# Compares the cookie with another. When there are many cookies with
# the same name for a URL, the value of the smallest must be used.
def <=>(other)
# RFC 6265 5.4
# Precedence: 1. longer path 2. older creation
(@name <=> other.name).nonzero? ||
(other.path.length <=> @path.length).nonzero? ||
(@created_at <=> other.created_at).nonzero? ||
@value <=> other.value
end
module_function
class << self
def new(cookie, *args)
return cookie if cookie.is_a?(self)
def quote(s)
return s unless s.match(RE_BAD_CHAR)
super
end
"\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
# Tests if +target_path+ is under +base_path+ as described in RFC
# 6265 5.1.4. +base_path+ must be an absolute path.
# +target_path+ may be empty, in which case it is treated as the
# root path.
#
# e.g.
#
# path_match?('/admin/', '/admin/index') == true
# path_match?('/admin/', '/Admin/index') == false
# path_match?('/admin/', '/admin/') == true
# path_match?('/admin/', '/admin') == false
#
# path_match?('/admin', '/admin') == true
# path_match?('/admin', '/Admin') == false
# path_match?('/admin', '/admins') == false
# path_match?('/admin', '/admin/') == true
# path_match?('/admin', '/admin/index') == true
def path_match?(base_path, target_path)
base_path.start_with?("/") || (return false)
# RFC 6265 5.1.4
bsize = base_path.size
tsize = target_path.size
return bsize == 1 if tsize.zero? # treat empty target_path as "/"
return false unless target_path.start_with?(base_path)
return true if bsize == tsize || base_path.end_with?("/")
target_path[bsize] == "/"
end
end
def initialize(arg, *attrs)
@created_at = Time.now
if attrs.empty?
attr_hash = Hash.try_convert(arg)
else
@name = arg
@value, attr_hash = attrs
attr_hash = Hash.try_convert(attr_hash)
end
attr_hash.each do |key, val|
key = key.downcase.tr("-", "_").to_sym unless key.is_a?(Symbol)
case key
when :domain, :path
__send__(:"#{key}=", val)
else
instance_variable_set(:"@#{key}", val)
end
end if attr_hash
@path ||= "/"
raise ArgumentError, "name must be specified" if @name.nil?
end
def expires
@expires || (@created_at && @max_age ? @created_at + @max_age : nil)
end
def expired?(time = Time.now)
return false unless expires
expires <= time
end
# Returns a string for use in the Cookie header, i.e. `name=value`
# or `name="value"`.
def cookie_value
"#{@name}=#{Scanner.quote(@value)}"
end
alias_method :to_s, :cookie_value
# Tests if it is OK to send this cookie to a given `uri`. A
# RuntimeError is raised if the cookie's domain is unknown.
def valid_for_uri?(uri)
uri = URI(uri)
# RFC 6265 5.4
return false if @secure && uri.scheme != "https"
acceptable_from_uri?(uri) && Cookie.path_match?(@path, uri.path)
end
private
# Tests if it is OK to accept this cookie if it is sent from a given
# URI/URL, `uri`.
def acceptable_from_uri?(uri)
uri = URI(uri)
host = DomainName.new(uri.host)
# RFC 6265 5.3
if host.hostname == @domain
true
elsif @for_domain # !host-only-flag
host.cookie_domain?(@domain_name)
else
@domain.nil?
end
end
module Scanner
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
module_function
def quote(s)
return s unless s.match(RE_BAD_CHAR)
"\"#{s.gsub(/([\\"])/, "\\\\\\1")}\""
end
end
end
end

250
lib/httpx/plugins/cookies/set_cookie_parser.rb
View File

@ -3,151 +3,139 @@
require "strscan"
require "time"
module HTTPX::Plugins::Cookies
module SetCookieParser
unless Regexp.method_defined?(:match?)
# If you wonder why this is there: the oauth feature uses a refinement to enhance the
# Regexp class locally with #match? , but this is never tested, because ActiveSupport
# monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
# :nocov:
module RegexpExtensions
refine(Regexp) do
def match?(*args)
!match(*args).nil?
end
end
end
using(RegexpExtensions)
# :nocov:
end
module HTTPX
module Plugins::Cookies
module SetCookieParser
using(RegexpExtensions) unless Regexp.method_defined?(:match?)
# Whitespace.
RE_WSP = /[ \t]+/.freeze
# Whitespace.
RE_WSP = /[ \t]+/.freeze
# A pattern that matches a cookie name or attribute name which may
# be empty, capturing trailing whitespace.
RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
# A pattern that matches a cookie name or attribute name which may
# be empty, capturing trailing whitespace.
RE_NAME = /(?!#{RE_WSP})[^,;\\"=]*/.freeze
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
RE_BAD_CHAR = /([\x00-\x20\x7F",;\\])/.freeze
# A pattern that matches the comma in a (typically date) value.
RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
# A pattern that matches the comma in a (typically date) value.
RE_COOKIE_COMMA = /,(?=#{RE_WSP}?#{RE_NAME}=)/.freeze
module_function
module_function
def scan_dquoted(scanner)
s = +""
until scanner.eos?
break if scanner.skip(/"/)
if scanner.skip(/\\/)
s << scanner.getch
elsif scanner.scan(/[^"\\]+/)
s << scanner.matched
end
end
s
end
def scan_value(scanner, comma_as_separator = false)
value = +""
until scanner.eos?
if scanner.scan(/[^,;"]+/)
value << scanner.matched
elsif scanner.skip(/"/)
# RFC 6265 2.2
# A cookie-value may be DQUOTE'd.
value << scan_dquoted(scanner)
elsif scanner.check(/;/)
break
elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
break
else
value << scanner.getch
end
end
value.rstrip!
value
end
def scan_name_value(scanner, comma_as_separator = false)
name = scanner.scan(RE_NAME)
name.rstrip! if name
if scanner.skip(/\=/)
value = scan_value(scanner, comma_as_separator)
else
scan_value(scanner, comma_as_separator)
value = nil
end
[name, value]
end
def call(set_cookie)
scanner = StringScanner.new(set_cookie)
# RFC 6265 4.1.1 & 5.2
until scanner.eos?
start = scanner.pos
len = nil
scanner.skip(RE_WSP)
name, value = scan_name_value(scanner, true)
value = nil if name.empty?
attrs = {}
def scan_dquoted(scanner)
s = +""
until scanner.eos?
if scanner.skip(/,/)
# The comma is used as separator for concatenating multiple
# values of a header.
len = (scanner.pos - 1) - start
break
elsif scanner.skip(/;/)
scanner.skip(RE_WSP)
break if scanner.skip(/"/)
aname, avalue = scan_name_value(scanner, true)
next if aname.empty? || value.nil?
aname.downcase!
case aname
when "expires"
# RFC 6265 5.2.1
(avalue &&= Time.httpdate(avalue)) || next
when "max-age"
# RFC 6265 5.2.2
next unless /\A-?\d+\z/.match?(avalue)
avalue = Integer(avalue)
when "domain"
# RFC 6265 5.2.3
# An empty value SHOULD be ignored.
next if avalue.nil? || avalue.empty?
when "path"
# RFC 6265 5.2.4
# A relative path must be ignored rather than normalizing it
# to "/".
next unless avalue.start_with?("/")
when "secure", "httponly"
# RFC 6265 5.2.5, 5.2.6
avalue = true
end
attrs[aname] = avalue
if scanner.skip(/\\/)
s << scanner.getch
elsif scanner.scan(/[^"\\]+/)
s << scanner.matched
end
end
len ||= scanner.pos - start
s
end
next if len > Cookie::MAX_LENGTH
def scan_value(scanner, comma_as_separator = false)
value = +""
yield(name, value, attrs) if name && !name.empty? && value
until scanner.eos?
if scanner.scan(/[^,;"]+/)
value << scanner.matched
elsif scanner.skip(/"/)
# RFC 6265 2.2
# A cookie-value may be DQUOTE'd.
value << scan_dquoted(scanner)
elsif scanner.check(/;/)
break
elsif comma_as_separator && scanner.check(RE_COOKIE_COMMA)
break
else
value << scanner.getch
end
end
value.rstrip!
value
end
def scan_name_value(scanner, comma_as_separator = false)
name = scanner.scan(RE_NAME)
name.rstrip! if name
if scanner.skip(/=/)
value = scan_value(scanner, comma_as_separator)
else
scan_value(scanner, comma_as_separator)
value = nil
end
[name, value]
end
def call(set_cookie)
scanner = StringScanner.new(set_cookie)
# RFC 6265 4.1.1 & 5.2
until scanner.eos?
start = scanner.pos
len = nil
scanner.skip(RE_WSP)
name, value = scan_name_value(scanner, true)
value = nil if name.empty?
attrs = {}
until scanner.eos?
if scanner.skip(/,/)
# The comma is used as separator for concatenating multiple
# values of a header.
len = (scanner.pos - 1) - start
break
elsif scanner.skip(/;/)
scanner.skip(RE_WSP)
aname, avalue = scan_name_value(scanner, true)
next if aname.empty? || value.nil?
aname.downcase!
case aname
when "expires"
# RFC 6265 5.2.1
(avalue &&= Time.httpdate(avalue)) || next
when "max-age"
# RFC 6265 5.2.2
next unless /\A-?\d+\z/.match?(avalue)
avalue = Integer(avalue)
when "domain"
# RFC 6265 5.2.3
# An empty value SHOULD be ignored.
next if avalue.nil? || avalue.empty?
when "path"
# RFC 6265 5.2.4
# A relative path must be ignored rather than normalizing it
# to "/".
next unless avalue.start_with?("/")
when "secure", "httponly"
# RFC 6265 5.2.5, 5.2.6
avalue = true
end
attrs[aname] = avalue
end
end
len ||= scanner.pos - start
next if len > Cookie::MAX_LENGTH
yield(name, value, attrs) if name && !name.empty? && value
end
end
end
end

2
lib/httpx/plugins/digest_authentication.rb
View File

@ -81,7 +81,7 @@ module HTTPX
uri = request.path
params = Hash[auth_info.split(/ *, */)
params = Hash[auth_info.split(/ *, */) # rubocop:disable Style/HashTransformValues
.map { |val| val.split("=") }
.map { |k, v| [k, v.delete("\"")] }]
nonce = params["nonce"]

7
lib/httpx/plugins/expect.rb
View File

@ -33,11 +33,8 @@ module HTTPX
super
return if @body.nil?
if (threshold = options.expect_threshold_size)
unless unbounded_body?
return if @body.bytesize < threshold
end
end
threshold = options.expect_threshold_size
return if threshold && !unbounded_body? && @body.bytesize < threshold
@headers["expect"] = "100-continue"
end

4
lib/httpx/plugins/proxy/socks4.rb
View File

@ -91,6 +91,8 @@ module HTTPX
end
module Packet
using(RegexpExtensions) unless Regexp.method_defined?(:match?)
module_function
def connect(parameters, uri)
@ -101,7 +103,7 @@ module HTTPX
packet << [ip.to_i].pack("N")
rescue IPAddr::InvalidAddressError
if parameters.uri.scheme =~ /^socks4a?$/
if /^socks4a?$/.match?(parameters.uri.scheme)
# resolv defaults to IPv4, and socks4 doesn't support IPv6 otherwise
ip = IPAddr.new(Resolv.getaddress(uri.host))
packet << [ip.to_i].pack("N")

2
lib/httpx/pool.rb
View File

@ -166,7 +166,6 @@ module HTTPX
resolver.on(:error, &method(:on_resolver_error))
resolver.on(:close) { on_resolver_close(resolver) }
resolver
# rubocop: disable Layout/RescueEnsureAlignment
rescue ArgumentError
# this block is here because of an error which happens on CI from time to time
warn "tried resolver: #{resolver_type}"
@ -174,7 +173,6 @@ module HTTPX
warn "new: #{resolver_type.method(:new).source_location}"
raise
end
# rubocop: enable Layout/RescueEnsureAlignment
end
end
end

12
lib/httpx/request.rb
View File

@ -33,9 +33,7 @@ module HTTPX
USER_AGENT = "httpx.rb/#{VERSION}"
attr_reader :verb, :uri, :headers, :body, :state
attr_reader :options, :response
attr_reader :verb, :uri, :headers, :body, :state, :options, :response
def_delegator :@body, :empty?
@ -43,7 +41,7 @@ module HTTPX
def initialize(verb, uri, options = {})
@verb = verb.to_s.downcase.to_sym
@uri = URI(uri)
@uri = Utils.uri(uri)
@options = Options.new(options)
raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
@ -63,13 +61,13 @@ module HTTPX
end
if RUBY_VERSION < "2.2"
# rubocop: disable Lint/UriEscapeUnescape:
URIParser = URI::DEFAULT_PARSER
def initialize_with_escape(verb, uri, options = {})
initialize_without_escape(verb, URI.escape(uri.to_s), options)
initialize_without_escape(verb, URIParser.escape(uri.to_s), options)
end
alias_method :initialize_without_escape, :initialize
alias_method :initialize, :initialize_with_escape
# rubocop: enable Lint/UriEscapeUnescape:
end
def merge_headers(h)

11
lib/httpx/resolver/https.rb
View File

@ -94,7 +94,14 @@ module HTTPX
def resolve(connection = @connections.first, hostname = nil)
return if @building_connection
hostname = hostname || @queries.key(connection) || connection.origin.host
hostname ||= @queries.key(connection)
if hostname.nil?
hostname = connection.origin.host
if hostname != connection.origin.non_ascii_hostname
log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" }
end
end
type = @_record_types[hostname].first
log { "resolver: query #{type} for #{hostname}" }
begin
@ -206,7 +213,7 @@ module HTTPX
case response.headers["content-type"]
when "application/dns-json",
"application/json",
%r{^application\/x\-javascript} # because google...
%r{^application/x-javascript} # because google...
payload = JSON.parse(response.to_s)
payload["Answer"]
when "application/dns-udpwireformat",

10
lib/httpx/resolver/native.rb
View File

@ -7,6 +7,7 @@ module HTTPX
class Resolver::Native
extend Forwardable
include Resolver::ResolverMixin
using URIExtensions
RESOLVE_TIMEOUT = 5
RECORD_TYPES = {
@ -237,7 +238,14 @@ module HTTPX
raise Error, "no URI to resolve" unless connection
return unless @write_buffer.empty?
hostname = hostname || @queries.key(connection) || connection.origin.host
hostname ||= @queries.key(connection)
if hostname.nil?
hostname = connection.origin.host
if hostname != connection.origin.non_ascii_hostname
log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" }
end
end
@queries[hostname] = connection
type = @_record_types[hostname].first
log { "resolver: query #{type} for #{hostname}" }

8
lib/httpx/response.rb
View File

@ -75,11 +75,11 @@ module HTTPX
@status == 204 ||
@status == 205 ||
@status == 304 || begin
content_length = @headers["content-length"]
return false if content_length.nil?
content_length = @headers["content-length"]
return false if content_length.nil?
content_length == "0"
end
content_length == "0"
end
end
class Body

6
lib/httpx/selector.rb
View File

@ -51,7 +51,7 @@ class HTTPX::Selector
READ_INTERESTS = %i[r rw].freeze
WRITE_INTERESTS = %i[w rw].freeze
def select_many(interval)
def select_many(interval, &block)
selectables, r, w = nil
# first, we group IOs based on interest type. On call to #interests however,
@ -102,9 +102,7 @@ class HTTPX::Selector
writers.delete(io)
end if readers
writers.each do |io|
yield io
end if writers
writers.each(&block) if writers
end
def select_one(interval)

15
lib/httpx/session.rb
View File

@ -66,7 +66,8 @@ module HTTPX
end
def find_connection(request, connections, options)
uri = URI(request.uri)
uri = request.uri
connection = pool.find_connection(uri, options) || build_connection(uri, options)
unless connections.nil? || connections.include?(connection)
connections << connection
@ -189,15 +190,13 @@ module HTTPX
begin
# guarantee ordered responses
loop do
begin
request = requests.first
pool.next_tick until (response = fetch_response(request, connections, request_options))
request = requests.first
pool.next_tick until (response = fetch_response(request, connections, request_options))
responses << response
requests.shift
responses << response
requests.shift
break if requests.empty? || pool.empty?
end
break if requests.empty? || pool.empty?
end
responses
ensure

27
lib/httpx/utils.rb
View File

@ -2,6 +2,8 @@
module HTTPX
module Utils
using URIExtensions
module_function
# The value of this field can be either an HTTP-date or a number of
@ -14,5 +16,30 @@ module HTTPX
time = Time.httpdate(retry_after)
time - Time.now
end
if RUBY_VERSION < "2.3"
def uri(*args)
URI(*args)
end
else
URIParser = URI::RFC2396_Parser.new
def uri(uri)
return Kernel.URI(uri) unless uri.is_a?(String) && !uri.ascii_only?
uri = Kernel.URI(URIParser.escape(uri))
non_ascii_hostname = URIParser.unescape(uri.host)
non_ascii_hostname.force_encoding(Encoding::UTF8) if RUBY_ENGINE == "jruby"
idna_hostname = DomainName.new(non_ascii_hostname).hostname
uri.host = idna_hostname
uri.non_ascii_hostname = non_ascii_hostname
uri
end
end
end
end

6
profiler/helpers.rb
View File

@ -91,10 +91,8 @@ module ProfilerHelpers
end
end
def memory_profile
def memory_profile(&block)
require "memory_profiler"
MemoryProfiler.report(allow_files: ["lib/httpx"]) do
yield
end.pretty_print
MemoryProfiler.report(allow_files: ["lib/httpx"], &block).pretty_print
end
end

17
sig/domain_name.rbs Normal file
View File

@ -0,0 +1,17 @@
module HTTPX
class DomainName
type domain = string | DomainName
include Comparable
def normalize: (String) -> String
def cookie_domain?: (domain, ?bool?) -> bool
def self.new: (domain) -> untyped
private
def initialize: (string) -> untyped
end
end

19
sig/plugins/cookies/domain_name.rbs
View File

@ -1,19 +0,0 @@
module HTTPX
module Plugins::Cookies
class DomainName
type domain = string | DomainName
include Comparable
def normalize: (String) -> String
def cookie_domain?: (domain, ?bool?) -> bool
def self.new: (domain) -> untyped
private
def initialize: (string) -> untyped
end
end
end

4
test/http_test.rb
View File

@ -40,8 +40,8 @@ class HTTPTest < Minitest::Test
assert log_output.match(/HEADER: Connection: close/)
# assert response headers
assert log_output.match(%r{HEADLINE: 200 HTTP/1\.1})
assert log_output.match(/HEADER: content\-type: \w+/)
assert log_output.match(/HEADER: content\-length: \d+/)
assert log_output.match(/HEADER: content-type: \w+/)
assert log_output.match(/HEADER: content-length: \d+/)
end
private

4
test/https_test.rb
View File

@ -70,8 +70,8 @@ class HTTPSTest < Minitest::Test
assert log_output.match(%r{HEADER: accept: */*})
# assert response headers
assert log_output.match(/HEADER: :status: 200/)
assert log_output.match(/HEADER: content\-type: \w+/)
assert log_output.match(/HEADER: content\-length: \d+/)
assert log_output.match(/HEADER: content-type: \w+/)
assert log_output.match(/HEADER: content-length: \d+/)
end
private

8
test/support/requests/get.rb
View File

@ -75,5 +75,13 @@ module Requests
request = response.instance_variable_get(:@request)
verify_header(request.headers, "accept", "text/html")
end
def test_get_non_ascii
response = HTTPX.get("http://bücher.ch")
verify_status(response, 200)
response = HTTPX.get(build_uri("/get?q=ã"))
verify_status(response, 200)
end unless RUBY_VERSION < "2.3"
end
end

2
test/support/requests/plugins/cookies.rb
View File

@ -192,7 +192,7 @@ module Requests
end
def cookies_set_uri(cookies)
build_uri("/cookies/set?" + URI.encode_www_form(cookies))
build_uri("/cookies/set?#{URI.encode_www_form(cookies)}")
end
def verify_cookies(jar, cookies)

2
test/support/requests/plugins/follow_redirects.rb
View File

@ -67,7 +67,7 @@ module Requests
private
def redirect_uri(redirect_uri = redirect_location)
build_uri("/redirect-to?url=" + redirect_uri)
build_uri("/redirect-to?url=#{redirect_uri}")
end
def max_redirect_uri(n)

4
test/support/requests/with_body.rb
View File

@ -88,8 +88,8 @@ module Requests
define_method :"test_#{meth}_multiple_params" do
uri = build_uri("/#{meth}")
response1, response2 = HTTPX.request([
[meth, uri, body: "data"],
[meth, uri, form: { "foo" => "bar" }],
[meth, uri, { body: "data" }],
[meth, uri, { form: { "foo" => "bar" } }],
], max_concurrent_requests: 1) # because httpbin sucks and can't handle pipeline requests
verify_status(response1, 200)