From ad1ff620b03d72c92e64ae36634f418d2da9fe49 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Jan 2018 23:04:53 +0000 Subject: [PATCH 01/18] finally fixed the digest auth; the real issue was lack of a cookie, which apparently httpbin needs --- lib/httpx/plugins/digest_authentication.rb | 42 ++++++------------- .../requests/plugins/authentication.rb | 2 +- 2 files changed, 13 insertions(+), 31 deletions(-) diff --git a/lib/httpx/plugins/digest_authentication.rb b/lib/httpx/plugins/digest_authentication.rb index e54fdac5..e87af96c 100644 --- a/lib/httpx/plugins/digest_authentication.rb +++ b/lib/httpx/plugins/digest_authentication.rb @@ -72,6 +72,7 @@ module HTTPX # TODO: assert if auth-type is Digest auth_info = www[/^(\w+) (.*)/, 2] + uri = request.path params = Hash[ auth_info.scan(/(\w+)="(.*?)"/) ] @@ -112,43 +113,24 @@ module HTTPX end ha1 = algorithm.hexdigest(a1) - ha2 = algorithm.hexdigest("#{method}:#{request.path}") - + ha2 = algorithm.hexdigest("#{method}:#{uri}") request_digest = [ha1, nonce] request_digest.push(nc, cnonce, qop) if qop request_digest << ha2 request_digest = request_digest.join(":") header = [ - "username=\"#{user}\"", - "response=\"#{algorithm.hexdigest(request_digest)}\"", - "uri=\"#{request.path}\"", - "nonce=\"#{nonce}\"" + %[username="#{user}"], + %[nonce="#{nonce}"], + %[uri="#{uri}"], + %[response="#{algorithm.hexdigest(request_digest)}"] ] - header << "realm=\"#{params["realm"]}\"" if params.key?("realm") - header << "opaque=\"#{params["opaque"]}\"" if params.key?("opaque") - header << "algorithm=#{params["algorithm"]}" if params.key?("algorithm") - header << "cnonce=#{cnonce}" if cnonce - header << "nc=#{nc}" - header << "qop=#{qop}" if qop - # - # if qop.nil? then - # elsif iis then - # "qop=\"#{qop}\"" - # else - # "qop=#{qop}" - # end, - # if qop then - # [ - # "nc=#{"%08x" % nonce}", - # "cnonce=\"#{cnonce}\"", - # ] - # end, - # if params.key?("opaque") then - # "opaque=\"#{params["opaque"]}\"" - # end - # ].compact - + header << %[realm="#{params["realm"]}"] if params.key?("realm") + header << %[algorithm=#{params["algorithm"]}"] if params.key?("algorithm") + header << %[opaque="#{params["opaque"]}"] if params.key?("opaque") + header << %[cnonce="#{cnonce}"] if cnonce + header << %[nc=#{nc}] + header << %[qop=#{qop}] if qop header.join ", " end diff --git a/test/support/requests/plugins/authentication.rb b/test/support/requests/plugins/authentication.rb index c5a84856..fee0f9ab 100644 --- a/test/support/requests/plugins/authentication.rb +++ b/test/support/requests/plugins/authentication.rb @@ -20,7 +20,7 @@ module Requests end def test_plugin_digest_authentication - client = HTTPX.plugin(:digest_authentication) + client = HTTPX.plugin(:digest_authentication).headers("cookie" => "fake=fake_value") response = client.digest_authentication(user, pass).get(digest_auth_uri) verify_status(response.status, 200) body = json_body(response) From 812cba204663a4ec8c0c0111bb33a70483108f28 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Jan 2018 23:20:05 +0000 Subject: [PATCH 02/18] digest: guarantee that multi-requests are sent sequentially, as the auth header has to be validated each time with metadata from previous response --- lib/httpx/plugins/digest_authentication.rb | 49 +++++++++------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/lib/httpx/plugins/digest_authentication.rb b/lib/httpx/plugins/digest_authentication.rb index e87af96c..e9e1ef83 100644 --- a/lib/httpx/plugins/digest_authentication.rb +++ b/lib/httpx/plugins/digest_authentication.rb @@ -12,9 +12,7 @@ module HTTPX module InstanceMethods def digest_authentication(user, password) - @_digest_auth_user = user - @_digest_auth_pass = password - @_digest = Digest.new + @_digest = Digest.new(user, password) self end alias :digest_auth :digest_authentication @@ -26,31 +24,22 @@ module HTTPX #@keep_open = true requests = __build_reqs(*args, **options) - responses = __send_reqs(*requests) + probe_request = requests.first + prev_response = __send_reqs(*probe_request).first - failed_requests = [] - failed_responses_ids = responses.each_with_index.map do |response, index| - next unless response.status == 401 - request = requests[index] + unless prev_response.status == 401 + raise Error, "request doesn't require authentication (status: #{prev_response})" + end - token = @_digest.generate_header(@_digest_auth_user, - @_digest_auth_pass, - request, - response) + probe_request.transition(:idle) + responses = [] + requests.each do |request| + token = @_digest.generate_header(request, prev_response) request.headers["authorization"] = "Digest #{token}" - request.transition(:idle) - - failed_requests << request - - index - end.compact - - return responses if failed_requests.empty? - - repeated_responses = __send_reqs(*failed_requests) - repeated_responses.each_with_index do |rep, index| - responses[index] = rep + response = __send_reqs(*request).first + responses << response + prev_response = response end return responses.first if responses.size == 1 responses @@ -61,11 +50,13 @@ module HTTPX end class Digest - def initialize + def initialize(user, password) + @user = user + @password = password @nonce = 0 end - def generate_header(user, password, request, response, iis = false) + def generate_header(request, response, iis = false) method = request.verb.to_s.upcase www = response.headers["www-authenticate"] @@ -104,12 +95,12 @@ module HTTPX end a1 = if sess then - [ algorithm.hexdigest("#{user}:#{params["realm"]}:#{password}"), + [ algorithm.hexdigest("#{@user}:#{params["realm"]}:#{@password}"), nonce, cnonce, ].join ":" else - "#{user}:#{params["realm"]}:#{password}" + "#{@user}:#{params["realm"]}:#{@password}" end ha1 = algorithm.hexdigest(a1) @@ -120,7 +111,7 @@ module HTTPX request_digest = request_digest.join(":") header = [ - %[username="#{user}"], + %[username="#{@user}"], %[nonce="#{nonce}"], %[uri="#{uri}"], %[response="#{algorithm.hexdigest(request_digest)}"] From 50adc95c466bc74617440d59a020011ce0871ec5 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Jan 2018 23:21:56 +0000 Subject: [PATCH 03/18] added backcompat mode to core request implementation --- lib/httpx/plugins/digest_authentication.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/httpx/plugins/digest_authentication.rb b/lib/httpx/plugins/digest_authentication.rb index e9e1ef83..3ba3605d 100644 --- a/lib/httpx/plugins/digest_authentication.rb +++ b/lib/httpx/plugins/digest_authentication.rb @@ -17,12 +17,9 @@ module HTTPX end alias :digest_auth :digest_authentication - def request(*args, **options) + def request(*args, keep_open: @keep_open, **options) return super unless @_digest begin - #keep_open = @keep_open - #@keep_open = true - requests = __build_reqs(*args, **options) probe_request = requests.first prev_response = __send_reqs(*probe_request).first @@ -44,7 +41,7 @@ module HTTPX return responses.first if responses.size == 1 responses ensure - #@keep_open = keep_open + close unless keep_open end end end From ba4f6d9b01bd56d74631f10cc881de364af32584 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Jan 2018 23:22:11 +0000 Subject: [PATCH 04/18] fixed inspects and logger --- lib/httpx/channel/http1.rb | 2 +- lib/httpx/request.rb | 4 ++++ lib/httpx/response.rb | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/httpx/channel/http1.rb b/lib/httpx/channel/http1.rb index 73405e7d..30a10cda 100644 --- a/lib/httpx/channel/http1.rb +++ b/lib/httpx/channel/http1.rb @@ -144,7 +144,7 @@ module HTTPX buffer.clear request.headers.each do |field, value| buffer << "#{capitalized(field)}: #{value}" << CRLF - log { "<- HEADER: #{buffer.chomp.inspect}" } + log { "<- HEADER: #{buffer.chomp}" } @buffer << buffer buffer.clear end diff --git a/lib/httpx/request.rb b/lib/httpx/request.rb index 7a92d3db..f911c689 100644 --- a/lib/httpx/request.rb +++ b/lib/httpx/request.rb @@ -90,6 +90,10 @@ module HTTPX nil end + def inspect + "#" + end + class Body class << self def new(*, options) diff --git a/lib/httpx/response.rb b/lib/httpx/response.rb index 5f73d880..938f35aa 100644 --- a/lib/httpx/response.rb +++ b/lib/httpx/response.rb @@ -47,6 +47,10 @@ module HTTPX ContentType.parse(@headers["content-type"]) end + def inspect + "#" + end + class Body def initialize(response, threshold_size: , window_size: 1 << 14) @response = response From 61c82a0e1d73df49a8f95a27b75efd919a399d31 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Jan 2018 23:22:32 +0000 Subject: [PATCH 05/18] reset response when resetting to idle in the request --- lib/httpx/request.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/httpx/request.rb b/lib/httpx/request.rb index f911c689..cee5556f 100644 --- a/lib/httpx/request.rb +++ b/lib/httpx/request.rb @@ -165,7 +165,7 @@ module HTTPX def transition(nextstate) case nextstate when :idle - + @response = nil when :headers return unless @state == :idle when :body From 518ed5280e32b60cabfcc78d4b44a8a8a9233087 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 00:18:43 +0000 Subject: [PATCH 06/18] moved deflate and gzip to separate file --- lib/httpx/plugins/compression.rb | 115 +---------------------- lib/httpx/plugins/compression/deflate.rb | 85 +++++++++++++++++ lib/httpx/plugins/compression/gzip.rb | 65 +++++++++++++ 3 files changed, 153 insertions(+), 112 deletions(-) create mode 100644 lib/httpx/plugins/compression/deflate.rb create mode 100644 lib/httpx/plugins/compression/gzip.rb diff --git a/lib/httpx/plugins/compression.rb b/lib/httpx/plugins/compression.rb index 196055e3..c8cbc3e8 100644 --- a/lib/httpx/plugins/compression.rb +++ b/lib/httpx/plugins/compression.rb @@ -5,8 +5,9 @@ module HTTPX module Compression ACCEPT_ENCODING = %w[gzip deflate].freeze - def self.load_dependencies(*) - require "zlib" + def self.configure(klass, *) + klass.plugin(:"compression/gzip") + klass.plugin(:"compression/deflate") end module RequestMethods @@ -78,116 +79,6 @@ module HTTPX end end - module GZIPTranscoder - class Encoder < CompressEncoder - def write(chunk) - @compressed_chunk = chunk - end - - private - - def compressed_chunk - compressed = @compressed_chunk - compressed - ensure - @compressed_chunk = nil - end - - def compress - return unless @buffer.size.zero? - @raw.rewind - begin - gzip = Zlib::GzipWriter.new(self) - - while chunk = @raw.read(16_384) - gzip.write(chunk) - gzip.flush - compressed = compressed_chunk - @buffer << compressed - yield compressed if block_given? - end - ensure - gzip.close - end - end - end - - module_function - - def encode(payload) - Encoder.new(payload) - end - - def decode(io) - Zlib::GzipReader.new(io, window_size: 32 + Zlib::MAX_WBITS) - end - end - - module DeflateTranscoder - class Encoder < CompressEncoder - private - - def compress - return unless @buffer.size.zero? - @raw.rewind - begin - deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION, - Zlib::MAX_WBITS, - Zlib::MAX_MEM_LEVEL, - Zlib::HUFFMAN_ONLY) - while chunk = @raw.read(16_384) - compressed = deflater.deflate(chunk) - @buffer << compressed - yield compressed if block_given? - end - last = deflater.finish - @buffer << last - yield last if block_given? - ensure - deflater.close - end - end - end - - module_function - - class Decoder - def initialize(io) - @io = io - @inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS) - @buffer = StringIO.new - end - - def rewind - @buffer.rewind - end - - def read(*args) - return @buffer.read(*args) if @io.eof? - chunk = @io.read(*args) - inflated_chunk = @inflater.inflate(chunk) - inflated_chunk << @inflater.finish if @io.eof? - @buffer << chunk - inflated_chunk - end - - def close - @io.close - @io.unlink if @io.respond_to?(:unlink) - @inflater.close - end - end - - def encode(payload) - Encoder.new(payload) - end - - def decode(io) - Decoder.new(io) - end - end - Transcoder.register "gzip", GZIPTranscoder - Transcoder.register "deflate", DeflateTranscoder end register_plugin :compression, Compression end diff --git a/lib/httpx/plugins/compression/deflate.rb b/lib/httpx/plugins/compression/deflate.rb new file mode 100644 index 00000000..69322c4a --- /dev/null +++ b/lib/httpx/plugins/compression/deflate.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module HTTPX + module Plugins + module Compression + module Deflate + + def self.load_dependencies(*) + require "stringio" + require "zlib" + end + + def self.configure(*) + Transcoder.register "deflate", DeflateTranscoder + end + + module DeflateTranscoder + class Encoder < CompressEncoder + private + + def compress + return unless @buffer.size.zero? + @raw.rewind + begin + deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION, + Zlib::MAX_WBITS, + Zlib::MAX_MEM_LEVEL, + Zlib::HUFFMAN_ONLY) + while chunk = @raw.read(16_384) + compressed = deflater.deflate(chunk) + @buffer << compressed + yield compressed if block_given? + end + last = deflater.finish + @buffer << last + yield last if block_given? + ensure + deflater.close + end + end + end + + module_function + + class Decoder + def initialize(io) + @io = io + @inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS) + @buffer = StringIO.new + end + + def rewind + @buffer.rewind + end + + def read(*args) + return @buffer.read(*args) if @io.eof? + chunk = @io.read(*args) + inflated_chunk = @inflater.inflate(chunk) + inflated_chunk << @inflater.finish if @io.eof? + @buffer << chunk + inflated_chunk + end + + def close + @io.close + @io.unlink if @io.respond_to?(:unlink) + @inflater.close + end + end + + def encode(payload) + Encoder.new(payload) + end + + def decode(io) + Decoder.new(io) + end + end + end + end + register_plugin :"compression/deflate", Compression::Deflate + end +end + diff --git a/lib/httpx/plugins/compression/gzip.rb b/lib/httpx/plugins/compression/gzip.rb new file mode 100644 index 00000000..e69f6a6e --- /dev/null +++ b/lib/httpx/plugins/compression/gzip.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module HTTPX + module Plugins + module Compression + module GZIP + + def self_load_dependencies(*) + require "zlib" + end + + def self.configure(*) + Transcoder.register "gzip", GZIPTranscoder + end + + module GZIPTranscoder + class Encoder < CompressEncoder + def write(chunk) + @compressed_chunk = chunk + end + + private + + def compressed_chunk + compressed = @compressed_chunk + compressed + ensure + @compressed_chunk = nil + end + + def compress + return unless @buffer.size.zero? + @raw.rewind + begin + gzip = Zlib::GzipWriter.new(self) + + while chunk = @raw.read(16_384) + gzip.write(chunk) + gzip.flush + compressed = compressed_chunk + @buffer << compressed + yield compressed if block_given? + end + ensure + gzip.close + end + end + end + + module_function + + def encode(payload) + Encoder.new(payload) + end + + def decode(io) + Zlib::GzipReader.new(io, window_size: 32 + Zlib::MAX_WBITS) + end + end + + end + end + register_plugin :"compression/gzip", Compression::GZIP + end +end From 284bb066630d9dcb6a64dc4876c32cb6891d79d4 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 19:23:39 +0000 Subject: [PATCH 07/18] added versioning to response (important to test h2c, but also to ensure which 1.x is responding) --- lib/httpx/channel/http1.rb | 5 ++++- lib/httpx/channel/http2.rb | 2 +- lib/httpx/response.rb | 7 ++++--- test/client_test.rb | 2 +- test/response_test.rb | 12 ++++++------ 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/httpx/channel/http1.rb b/lib/httpx/channel/http1.rb index 30a10cda..c53e774a 100644 --- a/lib/httpx/channel/http1.rb +++ b/lib/httpx/channel/http1.rb @@ -73,7 +73,10 @@ module HTTPX log(2) { "headers received" } headers = @options.headers_class.new(h) - response = @options.response_class.new(@requests.last, @parser.status_code, headers, @options) + response = @options.response_class.new(@requests.last, + @parser.status_code, + @parser.http_version.join("."), + headers, @options) log { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" } log { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") } diff --git a/lib/httpx/channel/http2.rb b/lib/httpx/channel/http2.rb index 82cda036..391bba76 100644 --- a/lib/httpx/channel/http2.rb +++ b/lib/httpx/channel/http2.rb @@ -59,7 +59,7 @@ module HTTPX end _, status = h.shift headers = @options.headers_class.new(h) - response = @options.response_class.new(request, status, headers, @options) + response = @options.response_class.new(request, status, "2.0", headers, @options) request.response = response @streams[request] = stream end diff --git a/lib/httpx/response.rb b/lib/httpx/response.rb index 938f35aa..f0e55bbf 100644 --- a/lib/httpx/response.rb +++ b/lib/httpx/response.rb @@ -9,7 +9,7 @@ module HTTPX class Response extend Forwardable - attr_reader :status, :headers, :body + attr_reader :status, :headers, :body, :version def_delegator :@body, :to_s @@ -21,8 +21,9 @@ module HTTPX def_delegator :@request, :uri - def initialize(request, status, headers, options = {}) - @options = Options.new(options) + def initialize(request, status, version, headers, options = {}) + @options = Options.new(options) + @version = version @request = request @status = Integer(status) @headers = @options.headers_class.new(headers) diff --git a/test/client_test.rb b/test/client_test.rb index 89f71d66..760c0316 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -24,7 +24,7 @@ class ClientTest < Minitest::Test assert request.headers.respond_to?(:foo), "headers methods haven't been added" assert request.headers.foo == "headers-foo", "headers method is unexpected" assert client.respond_to?(:response), "response constructor was added" - response = client.response(nil, 200, {}) + response = client.response(nil, 200, "2.0", {}) assert response.respond_to?(:foo), "response methods haven't been added" assert response.foo == "response-foo", "response method is unexpected" assert request.headers.respond_to?(:foo), "headers methods haven't been added" diff --git a/test/response_test.rb b/test/response_test.rb index 01475c7c..3468b241 100644 --- a/test/response_test.rb +++ b/test/response_test.rb @@ -6,9 +6,9 @@ class ResponseTest < Minitest::Test include HTTPX def test_response_status - r1 = Response.new(request, 200, {}) + r1 = Response.new(request, 200, "1.1", {}) assert r1.status == 200, "unexpected status code (#{r1.status})" - r2 = Response.new(request, "200", {}) + r2 = Response.new(request, "200", "1.1", {}) assert r2.status == 200, "unexpected status code (#{r2.status})" end @@ -24,7 +24,7 @@ class ResponseTest < Minitest::Test def test_response_body_to_s opts = { threshold_size: 1024 } - body1 = Response::Body.new(Response.new(request, 200, {}), opts) + body1 = Response::Body.new(Response.new(request, 200, "2.0", {}), opts) assert body1.empty?, "body must be empty after initialization" body1.write("foo") assert body1 == "foo", "body must be updated" @@ -32,7 +32,7 @@ class ResponseTest < Minitest::Test body1.write("bar") assert body1 == "foobar", "body must buffer subsequent chunks" - body3 = Response::Body.new(Response.new(request("head"), 200, {}), opts) + body3 = Response::Body.new(Response.new(request("head"), 200, "2.0", {}), opts) assert body3.empty?, "body must be empty after initialization" assert body3 == "", "HEAD requets body must be empty" @@ -40,7 +40,7 @@ class ResponseTest < Minitest::Test def test_response_body_each opts = { threshold_size: 1024 } - body1 = Response::Body.new(Response.new(request, 200, {}), opts) + body1 = Response::Body.new(Response.new(request, 200, "2.0", {}), opts) body1.write("foo") assert body1.each.to_a == %w(foo), "must yield buffer" body1.write("foo") @@ -59,6 +59,6 @@ class ResponseTest < Minitest::Test end def resource - @resource ||= Response.new(request, 200, {}) + @resource ||= Response.new(request, 200, "2.0", {}) end end From 85111a36a9a310feb4e4c999622309a81774f414 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 19:24:35 +0000 Subject: [PATCH 08/18] moved all response callbacks to the client, this will enable better plugin overwrites --- lib/httpx/client.rb | 19 +++++++++++++++++-- lib/httpx/connection.rb | 18 ++---------------- lib/httpx/plugins/proxy.rb | 6 +++--- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/lib/httpx/client.rb b/lib/httpx/client.rb index 7a7ba421..3e0493b9 100644 --- a/lib/httpx/client.rb +++ b/lib/httpx/client.rb @@ -7,6 +7,7 @@ module HTTPX def initialize(options = {}) @default_options = self.class.default_options.merge(options) @connection = Connection.new(@default_options) + @responses = {} if block_given? begin @keep_open = true @@ -33,10 +34,24 @@ module HTTPX private + def on_response(request, response) + @responses[request] = response + end + + def fetch_response(request) + response = @responses.delete(request) + if response.is_a?(ErrorResponse) && response.retryable? + channel = find_channel(request) + channel.send(request, retries: response.retries - 1) + return + end + response + end + def find_channel(request) uri = URI(request.uri) @connection.find_channel(uri) || - @connection.build_channel(uri) + @connection.build_channel(uri, &method(:on_response)) end def __build_reqs(*args, **options) @@ -71,7 +86,7 @@ module HTTPX # guarantee ordered responses loop do request = requests.shift - @connection.next_tick until response = @connection.response(request) + @connection.next_tick until response = fetch_response(request) responses << response diff --git a/lib/httpx/connection.rb b/lib/httpx/connection.rb index 2e3bf723..9cdbb4b3 100644 --- a/lib/httpx/connection.rb +++ b/lib/httpx/connection.rb @@ -10,7 +10,6 @@ module HTTPX @timeout = options.timeout @selector = Selector.new @channels = [] - @responses = {} end def running? @@ -39,17 +38,8 @@ module HTTPX end end - def response(request) - response = @responses.delete(request) - if response.is_a?(ErrorResponse) && response.retryable? - send(request, retries: response.retries - 1) - return - end - response - end - - def build_channel(uri) - channel = Channel.by(uri, @options, &method(:on_response)) + def build_channel(uri, &on_response) + channel = Channel.by(uri, @options, &on_response) register_channel(channel) channel end @@ -66,10 +56,6 @@ module HTTPX private - def on_response(request, response) - @responses[request] = response - end - def register_channel(channel) monitor = @selector.register(channel, :rw) monitor.value = channel diff --git a/lib/httpx/plugins/proxy.rb b/lib/httpx/plugins/proxy.rb index 950841f4..4ffd2191 100644 --- a/lib/httpx/plugins/proxy.rb +++ b/lib/httpx/plugins/proxy.rb @@ -53,7 +53,7 @@ module HTTPX uri = parameters.uri io = TCP.new(uri.host, uri.port, @default_options) proxy_type = Parameters.registry(parameters.uri.scheme) - channel = proxy_type.new(io, parameters, @default_options, &@connection.method(:on_response)) + channel = proxy_type.new(io, parameters, @default_options, &method(:on_response)) @connection.__send__(:register_channel, channel) channel end @@ -78,8 +78,8 @@ module HTTPX end class ProxyChannel < Channel - def initialize(io, parameters, options) - super(io, options) + def initialize(io, parameters, options, &blk) + super(io, options, &blk) @parameters = parameters @state = :idle end From 951504ae983ca2df0d780f5fbdc82463ed182535 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 19:25:05 +0000 Subject: [PATCH 09/18] socks proxies must also implement close --- lib/httpx/plugins/proxy/socks4.rb | 3 +++ lib/httpx/plugins/proxy/socks5.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/lib/httpx/plugins/proxy/socks4.rb b/lib/httpx/plugins/proxy/socks4.rb index c304930b..05e55711 100644 --- a/lib/httpx/plugins/proxy/socks4.rb +++ b/lib/httpx/plugins/proxy/socks4.rb @@ -67,6 +67,9 @@ module HTTPX @options = Options.new(options) end + def close + end + def consume(*) end diff --git a/lib/httpx/plugins/proxy/socks5.rb b/lib/httpx/plugins/proxy/socks5.rb index 6871762f..de26b846 100644 --- a/lib/httpx/plugins/proxy/socks5.rb +++ b/lib/httpx/plugins/proxy/socks5.rb @@ -95,6 +95,9 @@ module HTTPX @options = Options.new(options) end + def close + end + def consume(*) end From c55771906df7d3bfbf8f986db4868d90693a54bf Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 19:25:47 +0000 Subject: [PATCH 10/18] http/1: passing retries as well (not doing anything with them doh) --- lib/httpx/channel/http1.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/httpx/channel/http1.rb b/lib/httpx/channel/http1.rb index c53e774a..241cf640 100644 --- a/lib/httpx/channel/http1.rb +++ b/lib/httpx/channel/http1.rb @@ -12,6 +12,7 @@ module HTTPX def initialize(buffer, options) @options = Options.new(options) @max_concurrent_requests = @options.max_concurrent_requests + @retries = options.max_retries @parser = HTTP::Parser.new(self) @parser.header_value_type = :arrays @buffer = buffer @@ -34,7 +35,7 @@ module HTTPX @parser << data end - def send(request, **) + def send(request, retries: @retries, **) if @requests.size >= @max_concurrent_requests @pending << request return From 7f22b3f39857dedd272bf70e877d7d1a277e68b3 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Sat, 6 Jan 2018 19:26:02 +0000 Subject: [PATCH 11/18] added a new socks5 proxy --- test/support/requests/plugins/proxy.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/requests/plugins/proxy.rb b/test/support/requests/plugins/proxy.rb index 568b6991..5c69da5d 100644 --- a/test/support/requests/plugins/proxy.rb +++ b/test/support/requests/plugins/proxy.rb @@ -55,7 +55,7 @@ module Requests end def socks5_proxy_uri - "socks5://37.59.56.88:13372" + "socks5://118.201.230.192:58303" end end end From 491bf11e2e7ee9ae2a67c7fb2412ed297146525e Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Mon, 8 Jan 2018 22:28:39 +0000 Subject: [PATCH 12/18] http1: allow for callback to be called outside of the HTTP::Parser callback-scope (as info only loads later, like #upgrade_data) --- lib/httpx/channel/http1.rb | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/httpx/channel/http1.rb b/lib/httpx/channel/http1.rb index 241cf640..1534b5c9 100644 --- a/lib/httpx/channel/http1.rb +++ b/lib/httpx/channel/http1.rb @@ -19,10 +19,12 @@ module HTTPX @version = [1,1] @pending = [] @requests = [] + @has_response = false end def close - @parser.reset! + @parser.reset! + @has_response = false end def empty? @@ -33,6 +35,7 @@ module HTTPX def <<(data) @parser << data + dispatch if @has_response end def send(request, retries: @retries, **) @@ -84,8 +87,8 @@ module HTTPX request.response = response # parser can't say if it's parsing GET or HEAD, # call the completeness callback manually - on_message_complete if request.verb == :head || - request.verb == :connect + dispatch if request.verb == :head || + request.verb == :connect end def on_body(chunk) @@ -96,7 +99,10 @@ module HTTPX def on_message_complete log(2) { "parsing complete" } - @parser.reset! + @has_response = true + end + + def dispatch request = @requests.first return handle(request) if request.expects? @@ -104,6 +110,11 @@ module HTTPX response = request.response emit(:response, request, response) + if @parser.upgrade? + response << @parser.upgrade_data + throw(:called) + end + close send(@pending.shift) unless @pending.empty? if response.headers["connection"] == "close" unless @requests.empty? @@ -130,6 +141,7 @@ module HTTPX end def handle(request) + @has_response = false set_request_headers(request) catch(:buffer_full) do request.transition(:headers) From 30fcdfeb2c23771b92d6c87053a2d22a1e2b5476 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Mon, 8 Jan 2018 22:29:14 +0000 Subject: [PATCH 13/18] http2: added http2_settings as configurable options --- lib/httpx/channel/http2.rb | 2 +- test/options_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/httpx/channel/http2.rb b/lib/httpx/channel/http2.rb index 391bba76..c36b62cd 100644 --- a/lib/httpx/channel/http2.rb +++ b/lib/httpx/channel/http2.rb @@ -109,7 +109,7 @@ module HTTPX end def init_connection - @connection = HTTP2::Client.new(settings_enable_push: 0) + @connection = HTTP2::Client.new(@options.http2_settings) @connection.on(:frame, &method(:on_frame)) @connection.on(:frame_sent, &method(:on_frame_sent)) @connection.on(:frame_received, &method(:on_frame_received)) diff --git a/test/options_test.rb b/test/options_test.rb index fa2342f5..9bb71b20 100644 --- a/test/options_test.rb +++ b/test/options_test.rb @@ -67,6 +67,7 @@ class OptionsSpec < Minitest::Test :form => {:bar => "bar"}, :timeout => Timeout::PerOperation.new, :ssl => {:foo => "bar", :alpn_protocols => %w[h2 http/1.1] }, + :http2_settings => { :settings_enable_push => 0 }, :fallback_protocol => "http/1.1", :headers => {"Foo" => "foo", "Accept" => "xml", "Bar" => "bar"}, :max_concurrent_requests => 100, From 0dc00033330a16f19c6af6ddbaaf6186cdc06636 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Mon, 8 Jan 2018 22:29:39 +0000 Subject: [PATCH 14/18] http2: added http2_settings as configurable options; allow parser to be upgraded --- lib/httpx/channel.rb | 19 +++++++++++++------ lib/httpx/channel/http2.rb | 8 ++++++++ lib/httpx/options.rb | 7 +++---- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/lib/httpx/channel.rb b/lib/httpx/channel.rb index 4459675a..5f1008ef 100644 --- a/lib/httpx/channel.rb +++ b/lib/httpx/channel.rb @@ -115,6 +115,11 @@ module HTTPX nil end + def upgrade_parser(protocol) + @parser.close if @parser + @parser = build_parser(protocol) + end + private def connect @@ -151,12 +156,14 @@ module HTTPX end def parser - @parser || begin - @parser = registry(@io.protocol).new(@write_buffer, @options) - @parser.on(:response, &@on_response) - @parser.on(:close) { throw(:close, self) } - @parser - end + @parser ||= build_parser + end + + def build_parser(protocol=@io.protocol) + parser = registry(protocol).new(@write_buffer, @options) + parser.on(:response, &@on_response) + parser.on(:close) { throw(:close, self) } + parser end end end diff --git a/lib/httpx/channel/http2.rb b/lib/httpx/channel/http2.rb index c36b62cd..e60f00e5 100644 --- a/lib/httpx/channel/http2.rb +++ b/lib/httpx/channel/http2.rb @@ -202,6 +202,14 @@ module HTTPX stream.refuse # TODO: policy for handling promises end + + def method_missing(meth, *args, &blk) + if @connection.respond_to?(meth) + @connection.__send__(meth, *args, &blk) + else + super + end + end end Channel.register "h2", Channel::HTTP2 end diff --git a/lib/httpx/options.rb b/lib/httpx/options.rb index 178c1cbe..6bc0b562 100644 --- a/lib/httpx/options.rb +++ b/lib/httpx/options.rb @@ -42,6 +42,7 @@ module HTTPX :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil, :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i, :ssl => { alpn_protocols: %w[h2 http/1.1] }, + :http2_settings => { settings_enable_push: 0 }, :fallback_protocol => "http/1.1", :timeout => Timeout.by(:per_operation), :headers => {}, @@ -84,7 +85,7 @@ module HTTPX %w[ params form json body - follow ssl max_retries + follow ssl http2_settings max_retries request_class response_class headers_class response_body_class io fallback_protocol debug debug_level ].each do |method_name| @@ -97,9 +98,7 @@ module HTTPX merged = h1.merge(h2) do |k, v1, v2| case k - when :headers - v1.merge(v2) - when :ssl + when :headers, :ssl, :http2_settings v1.merge(v2) else v2 From e762faedfa965d659300f47b9bfdc3371bf9b56c Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Mon, 8 Jan 2018 22:30:50 +0000 Subject: [PATCH 15/18] added h2c plugin --- lib/httpx/plugins/h2c.rb | 109 +++++++++++++++++++++++++++ test/http1_test.rb | 1 + test/support/requests/plugins/h2c.rb | 22 ++++++ 3 files changed, 132 insertions(+) create mode 100644 lib/httpx/plugins/h2c.rb create mode 100644 test/support/requests/plugins/h2c.rb diff --git a/lib/httpx/plugins/h2c.rb b/lib/httpx/plugins/h2c.rb new file mode 100644 index 00000000..1bafb2fe --- /dev/null +++ b/lib/httpx/plugins/h2c.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +module HTTPX + module Plugins + module H2C + def self.load_dependencies(*) + require "base64" + end + + module InstanceMethods + def request(*args, keep_open: @keep_open, **options) + return super if @_h2c_probed + begin + requests = __build_reqs(*args, **options) + + upgrade_request = requests.first + return super unless valid_h2c_upgrade_request?(upgrade_request) + upgrade_request.headers["upgrade"] = "h2c" + upgrade_request.headers["http2-settings"] = FrameBuilder.settings_value(@default_options.http2_settings) + # TODO: validate! + upgrade_response = __send_reqs(*upgrade_request).first + + if upgrade_response.status == 101 + channel = find_channel(upgrade_request) + parser = channel.upgrade_parser("h2") + parser.extend(UpgradeExtensions) + parser.upgrade(upgrade_request, upgrade_response, **options) + data = upgrade_response.to_s + parser << data + responses = __send_reqs(*requests) + else + # proceed as usual + responses = [upgrade_response] + __send_reqs(*requests[1..-1]) + end + return responses.first if responses.size == 1 + responses + ensure + @_h2c_probed = true + close unless keep_open + end + end + + private + + VALID_H2C_METHODS = %i[get options head].freeze + private_constant :VALID_H2C_METHODS + + def valid_h2c_upgrade_request?(request) + VALID_H2C_METHODS.include?(request.verb) && + request.scheme == "http" + end + end + + module UpgradeExtensions + def upgrade(request, response, retries: @retries, **) + @connection.send_connection_preface + # skip checks, it is assumed that this is the first + # request in the connection + stream = @connection.new_stream + # Stream 1 is implicitly "half-closed" from the client toward the server (see Section 5.1) + stream.__send__(:event, :half_closed_local) + stream.on(:close) do |error| + if request.expects? + return handle(request, stream) + end + response = request.response || ErrorResponse.new(error, retries) + emit(:response, request, response) + log(2, "#{stream.id}: ") { "closing stream" } + + + @streams.delete(request) + send(@pending.shift) unless @pending.empty? + end + stream.on(:half_close) do + log(2, "#{stream.id}: ") { "waiting for response..." } + end + # stream.on(:altsvc) + stream.on(:headers) do |h| + log(stream.id) do + h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n") + end + _, status = h.shift + headers = @options.headers_class.new(h) + response = @options.response_class.new(request, status, "2.0", headers, @options) + request.response = response + @streams[request] = stream + end + stream.on(:data) do |data| + log(1, "#{stream.id}: ") { "<- DATA: #{data.bytesize} bytes..." } + log(2, "#{stream.id}: ") { "<- #{data.inspect}" } + request.response << data + end + @streams[request] = stream + end + end + + module FrameBuilder + include HTTP2 + module_function + + def settings_value(settings) + frame = Framer.new.generate(type: :settings, stream: 0, payload: settings) + Base64.urlsafe_encode64(frame[9..-1]) + end + end + end + register_plugin(:h2c, H2C) + end +end diff --git a/test/http1_test.rb b/test/http1_test.rb index b6cf3156..afcd36ce 100644 --- a/test/http1_test.rb +++ b/test/http1_test.rb @@ -18,6 +18,7 @@ class HTTP1Test < HTTPTest include Plugins::FollowRedirects include Plugins::Cookies include Plugins::Compression + include Plugins::H2C private diff --git a/test/support/requests/plugins/h2c.rb b/test/support/requests/plugins/h2c.rb new file mode 100644 index 00000000..f009dd27 --- /dev/null +++ b/test/support/requests/plugins/h2c.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Requests + module Plugins + module H2C + def test_plugin_h2c_disabled + uri = build_uri("/get") + response = HTTPX.get(uri) + verify_status(response.status, 200) + assert response.version == "1.1", "http requests should be by default in HTTP/1.1" + end + + def test_plugin_h2c + client = HTTPX.plugin(:h2c) + uri = build_uri("/get") + response = client.get(uri) + verify_status(response.status, 200) + assert response.version == "2.0", "http h2c requests should be in HTTP/2" + end + end + end +end From d7fd96763ef44bfe7ba084f92bd37fc818a8fcfa Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Tue, 9 Jan 2018 21:41:15 +0000 Subject: [PATCH 16/18] added missing upgrade, HTTP2-Settings in connection for h2c flow --- lib/httpx/channel/http1.rb | 7 ++++++- lib/httpx/plugins/h2c.rb | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/httpx/channel/http1.rb b/lib/httpx/channel/http1.rb index 1534b5c9..d0ee7962 100644 --- a/lib/httpx/channel/http1.rb +++ b/lib/httpx/channel/http1.rb @@ -178,8 +178,13 @@ module HTTPX end end + UPCASED = { + "www-authenticate" => "WWW-Authenticate", + "http2-settings" => "HTTP2-Settings" + } + def capitalized(field) - field.to_s.split("-").map(&:capitalize).join("-") + UPCASED[field] || field.to_s.split("-").map(&:capitalize).join("-") end end Channel.register "http/1.1", Channel::HTTP1 diff --git a/lib/httpx/plugins/h2c.rb b/lib/httpx/plugins/h2c.rb index 1bafb2fe..a750a869 100644 --- a/lib/httpx/plugins/h2c.rb +++ b/lib/httpx/plugins/h2c.rb @@ -16,6 +16,8 @@ module HTTPX upgrade_request = requests.first return super unless valid_h2c_upgrade_request?(upgrade_request) upgrade_request.headers["upgrade"] = "h2c" + upgrade_request.headers.add("connection", "upgrade") + upgrade_request.headers.add("connection", "http2-settings") upgrade_request.headers["http2-settings"] = FrameBuilder.settings_value(@default_options.http2_settings) # TODO: validate! upgrade_response = __send_reqs(*upgrade_request).first From 25925f95e64db75f74fbc1db7e165cc2fda7edd2 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Tue, 9 Jan 2018 21:41:41 +0000 Subject: [PATCH 17/18] added extra space in comma headers --- lib/httpx/headers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/httpx/headers.rb b/lib/httpx/headers.rb index f2c27858..644582f9 100644 --- a/lib/httpx/headers.rb +++ b/lib/httpx/headers.rb @@ -95,7 +95,7 @@ module HTTPX def each return enum_for(__method__) { @headers.size } unless block_given? @headers.each do |field, value| - yield(field, value.join(",")) unless value.empty? + yield(field, value.join(", ")) unless value.empty? end end From bb6bc00280ac5d9d72c551e716c3cbb24e6d5883 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Tue, 9 Jan 2018 21:52:20 +0000 Subject: [PATCH 18/18] setting stream callbacks in the same method --- lib/httpx/channel/http2.rb | 66 ++++++++++++++++++++------------------ lib/httpx/plugins/h2c.rb | 38 ++-------------------- 2 files changed, 38 insertions(+), 66 deletions(-) diff --git a/lib/httpx/channel/http2.rb b/lib/httpx/channel/http2.rb index e60f00e5..11619610 100644 --- a/lib/httpx/channel/http2.rb +++ b/lib/httpx/channel/http2.rb @@ -37,37 +37,7 @@ module HTTPX end unless stream = @streams[request] stream = @connection.new_stream - stream.on(:close) do |error| - if request.expects? - return handle(request, stream) - end - response = request.response || ErrorResponse.new(error, retries) - emit(:response, request, response) - log(2, "#{stream.id}: ") { "closing stream" } - - - @streams.delete(request) - send(@pending.shift) unless @pending.empty? - end - stream.on(:half_close) do - log(2, "#{stream.id}: ") { "waiting for response..." } - end - # stream.on(:altsvc) - stream.on(:headers) do |h| - log(stream.id) do - h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n") - end - _, status = h.shift - headers = @options.headers_class.new(h) - response = @options.response_class.new(request, status, "2.0", headers, @options) - request.response = response - @streams[request] = stream - end - stream.on(:data) do |data| - log(1, "#{stream.id}: ") { "<- DATA: #{data.bytesize} bytes..." } - log(2, "#{stream.id}: ") { "<- #{data.inspect}" } - request.response << data - end + handle_stream(stream, request) @streams[request] = stream end handle(request, stream) @@ -119,6 +89,40 @@ module HTTPX @connection.on(:goaway, &method(:on_close)) end + def handle_stream(stream, request) + stream.on(:close) do |error| + if request.expects? + return handle(request, stream) + end + response = request.response || ErrorResponse.new(error, retries) + emit(:response, request, response) + log(2, "#{stream.id}: ") { "closing stream" } + + + @streams.delete(request) + send(@pending.shift) unless @pending.empty? + end + stream.on(:half_close) do + log(2, "#{stream.id}: ") { "waiting for response..." } + end + # stream.on(:altsvc) + stream.on(:headers) do |h| + log(stream.id) do + h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n") + end + _, status = h.shift + headers = @options.headers_class.new(h) + response = @options.response_class.new(request, status, "2.0", headers, @options) + request.response = response + @streams[request] = stream + end + stream.on(:data) do |data| + log(1, "#{stream.id}: ") { "<- DATA: #{data.bytesize} bytes..." } + log(2, "#{stream.id}: ") { "<- #{data.inspect}" } + request.response << data + end + end + def join_headers(stream, request) set_request_headers(request) headers = {} diff --git a/lib/httpx/plugins/h2c.rb b/lib/httpx/plugins/h2c.rb index a750a869..afa67cc6 100644 --- a/lib/httpx/plugins/h2c.rb +++ b/lib/httpx/plugins/h2c.rb @@ -18,7 +18,7 @@ module HTTPX upgrade_request.headers["upgrade"] = "h2c" upgrade_request.headers.add("connection", "upgrade") upgrade_request.headers.add("connection", "http2-settings") - upgrade_request.headers["http2-settings"] = FrameBuilder.settings_value(@default_options.http2_settings) + upgrade_request.headers["http2-settings"] = HTTP2::Client.settings_header(@default_options.http2_settings) # TODO: validate! upgrade_response = __send_reqs(*upgrade_request).first @@ -58,40 +58,8 @@ module HTTPX @connection.send_connection_preface # skip checks, it is assumed that this is the first # request in the connection - stream = @connection.new_stream - # Stream 1 is implicitly "half-closed" from the client toward the server (see Section 5.1) - stream.__send__(:event, :half_closed_local) - stream.on(:close) do |error| - if request.expects? - return handle(request, stream) - end - response = request.response || ErrorResponse.new(error, retries) - emit(:response, request, response) - log(2, "#{stream.id}: ") { "closing stream" } - - - @streams.delete(request) - send(@pending.shift) unless @pending.empty? - end - stream.on(:half_close) do - log(2, "#{stream.id}: ") { "waiting for response..." } - end - # stream.on(:altsvc) - stream.on(:headers) do |h| - log(stream.id) do - h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n") - end - _, status = h.shift - headers = @options.headers_class.new(h) - response = @options.response_class.new(request, status, "2.0", headers, @options) - request.response = response - @streams[request] = stream - end - stream.on(:data) do |data| - log(1, "#{stream.id}: ") { "<- DATA: #{data.bytesize} bytes..." } - log(2, "#{stream.id}: ") { "<- #{data.inspect}" } - request.response << data - end + stream = @connection.upgrade + handle_stream(stream, request) @streams[request] = stream end end