From df26c97e3856628892dc5ff8054c212c8fdfdaa7 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Wed, 29 Mar 2023 23:00:50 +0100 Subject: [PATCH 1/3] changed http based tests, added #scheme, using it to correctly set no proxy domain --- test/http_test.rb | 4 ++-- test/https_test.rb | 4 ++-- test/support/http_helpers.rb | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/http_test.rb b/test/http_test.rb index 5d51ddc8..2f475e65 100644 --- a/test/http_test.rb +++ b/test/http_test.rb @@ -100,7 +100,7 @@ class HTTPTest < Minitest::Test private - def origin(orig = httpbin) - "http://#{orig}" + def scheme + "http://" end end diff --git a/test/https_test.rb b/test/https_test.rb index e3792952..186a55dd 100644 --- a/test/https_test.rb +++ b/test/https_test.rb @@ -180,7 +180,7 @@ class HTTPSTest < Minitest::Test private - def origin(orig = httpbin) - "https://#{orig}" + def scheme + "https://" end end diff --git a/test/support/http_helpers.rb b/test/support/http_helpers.rb index fe1e878e..8575664d 100644 --- a/test/support/http_helpers.rb +++ b/test/support/http_helpers.rb @@ -23,6 +23,10 @@ module HTTPHelpers end def httpbin_no_proxy - URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "httpbin.org")) + URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "#{scheme}httpbin.org")) + end + + def origin(orig = httpbin) + "#{scheme}#{orig}" end end From cbd695fb9c546dbb5fd56f34ee40c2279f36421a Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Wed, 29 Mar 2023 23:01:28 +0100 Subject: [PATCH 2/3] fix proxy discovery using proxy env vars connections weren't being correctly initiated, as proxies were filtered for the whole session based on URI.find_proxy for the first call. This fixes it by: * applying it to all used uris; * falling back to proxy options instead; * apply no_proxy option in case it's used, using `URI::Generic.use_proxy? --- lib/httpx/plugins/proxy.rb | 51 ++++++++++++++++-------------- sig/plugins/proxy.rbs | 6 +--- standalone_tests/env_proxy_test.rb | 22 +++++++++++-- 3 files changed, 49 insertions(+), 30 deletions(-) diff --git a/lib/httpx/plugins/proxy.rb b/lib/httpx/plugins/proxy.rb index 5d513733..ec792fab 100644 --- a/lib/httpx/plugins/proxy.rb +++ b/lib/httpx/plugins/proxy.rb @@ -94,34 +94,39 @@ module HTTPX module InstanceMethods private - def proxy_uris(uri, options) - @_proxy_uris ||= begin - uris = options.proxy ? Array(options.proxy[:uri]) : [] - if uris.empty? - uri = URI(uri).find_proxy - uris << uri if uri - end - uris - end - return if @_proxy_uris.empty? - - proxy = options.proxy - - return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty? - - proxy_opts = { uri: @_proxy_uris.first } - proxy_opts = options.proxy.merge(proxy_opts) if options.proxy - proxy_opts - end - def find_connection(request, connections, options) return super unless options.respond_to?(:proxy) uri = URI(request.uri) - next_proxy = proxy_uris(uri, options) - raise Error, "Failed to connect to proxy" unless next_proxy - proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host + proxy_opts = if (next_proxy = uri.find_proxy) + { uri: next_proxy } + else + proxy = options.proxy + + return super unless proxy + + return super(request, connections, options.merge(proxy: nil)) unless proxy.key?(:uri) + + @_proxy_uris ||= Array(proxy[:uri]) + + next_proxy = @_proxy_uris.first + raise Error, "Failed to connect to proxy" unless next_proxy + + if proxy.key?(:no_proxy) + next_proxy = URI(next_proxy) + + no_proxy = proxy[:no_proxy] + no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array) + + return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host, + next_proxy.port, no_proxy) + end + + proxy.merge(uri: next_proxy) + end + + proxy = Parameters.new(**proxy_opts) proxy_options = options.merge(proxy: proxy) connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options) diff --git a/sig/plugins/proxy.rbs b/sig/plugins/proxy.rbs index 9bfd41fa..35e92739 100644 --- a/sig/plugins/proxy.rbs +++ b/sig/plugins/proxy.rbs @@ -35,11 +35,7 @@ module HTTPX def self.extra_options: (Options) -> (Options & _ProxyOptions) module InstanceMethods - private - - def proxy_uris: (generic_uri, Options & _ProxyOptions) -> { uri: generic_uri, username: String, password: String } - | (generic_uri, Options & _ProxyOptions) -> { uri: generic_uri } - | (generic_uri, Options & _ProxyOptions) -> nil + @__proxy_uris: Array[generic_uri] end end diff --git a/standalone_tests/env_proxy_test.rb b/standalone_tests/env_proxy_test.rb index 521e494b..91f9cebc 100644 --- a/standalone_tests/env_proxy_test.rb +++ b/standalone_tests/env_proxy_test.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true HTTP_PROXY = ENV["HTTPX_HTTP_PROXY"] -ENV["HTTP_PROXY"] = HTTP_PROXY +ENV["http_proxy"] = HTTP_PROXY HTTPS_PROXY = ENV["HTTPX_HTTPS_PROXY"] -ENV["HTTPS_PROXY"] = HTTPS_PROXY +ENV["https_proxy"] = HTTPS_PROXY +NO_PROXY = ENV["HTTPBIN_NO_PROXY_HOST"] +ENV["no_proxy"] = URI(NO_PROXY).authority require "test_helper" require "support/http_helpers" @@ -13,6 +15,22 @@ class EnvProxyTest < Minitest::Test include HTTPHelpers using HTTPX::URIExtensions + def test_plugin_http_no_proxy + HTTPX.plugin(SessionWithPool).plugin(ProxyResponseDetector).wrap do |session| + # proxy + response = session.get("https://#{httpbin}/get") + verify_status(response, 200) + verify_body_length(response) + assert response.proxied? + + # no proxy + no_proxy_response = session.get("#{NO_PROXY}/get") + verify_status(no_proxy_response, 200) + verify_body_length(no_proxy_response) + assert !no_proxy_response.proxied? + end + end + def test_env_proxy_coalescing HTTPX.plugin(SessionWithPool).wrap do |session| response = session.get("https://#{httpbin}/get") From 7f59b9498e89d4a9eb6cf3340131ffd0e9a0e09e Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Thu, 30 Mar 2023 23:04:48 +0100 Subject: [PATCH 3/3] backport URI::Generic.use_proxy? for older rubies --- lib/httpx/plugins/proxy.rb | 49 ++++++++++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/lib/httpx/plugins/proxy.rb b/lib/httpx/plugins/proxy.rb index ec792fab..0aa05b43 100644 --- a/lib/httpx/plugins/proxy.rb +++ b/lib/httpx/plugins/proxy.rb @@ -18,6 +18,43 @@ module HTTPX Error = HTTPProxyError PROXY_ERRORS = [TimeoutError, IOError, SystemCallError, Error].freeze + class << self + def configure(klass) + klass.plugin(:"proxy/http") + klass.plugin(:"proxy/socks4") + klass.plugin(:"proxy/socks5") + end + + if URI::Generic.methods.include?(:use_proxy?) + def use_proxy?(*args) + URI::Generic.use_proxy?(*args) + end + else + # https://github.com/ruby/uri/blob/ae07f956a4bea00b4f54a75bd40b8fa918103eed/lib/uri/generic.rb + def use_proxy?(hostname, addr, port, no_proxy) + hostname = hostname.downcase + dothostname = ".#{hostname}" + no_proxy.scan(/([^:,\s]+)(?::(\d+))?/) do |p_host, p_port| + if !p_port || port == p_port.to_i + if p_host.start_with?(".") + return false if hostname.end_with?(p_host.downcase) + else + return false if dothostname.end_with?(".#{p_host.downcase}") + end + if addr + begin + return false if IPAddr.new(p_host).include?(addr) + rescue IPAddr::InvalidAddressError + next + end + end + end + end + true + end + end + end + class Parameters attr_reader :uri, :username, :password, :scheme @@ -77,14 +114,6 @@ module HTTPX end end - class << self - def configure(klass) - klass.plugin(:"proxy/http") - klass.plugin(:"proxy/socks4") - klass.plugin(:"proxy/socks5") - end - end - module OptionsMethods def option_proxy(value) value.is_a?(Parameters) ? value : Hash[value] @@ -119,8 +148,8 @@ module HTTPX no_proxy = proxy[:no_proxy] no_proxy = no_proxy.join(",") if no_proxy.is_a?(Array) - return super(request, connections, options.merge(proxy: nil)) unless URI::Generic.use_proxy?(uri.host, next_proxy.host, - next_proxy.port, no_proxy) + return super(request, connections, options.merge(proxy: nil)) unless Proxy.use_proxy?(uri.host, next_proxy.host, + next_proxy.port, no_proxy) end proxy.merge(uri: next_proxy)