diff --git a/Gemfile b/Gemfile index 20896089..5d30c31c 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,11 @@ group :test do gem "ruby-ntlm" gem "sentry-ruby" if RUBY_VERSION >= "2.4" gem "spy" - gem "webmock" + if RUBY_VERSION < "2.3.0" + gem "webmock", "< 3.15.0" + else + gem "webmock" + end gem "websocket-driver" gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2" @@ -115,8 +119,8 @@ group :assorted do if RUBY_VERSION < "2.2" gem "pry-byebug", "~> 3.4.3" else - gem "pry-byebug" gem "debug" if RUBY_VERSION >= "3.1.0" + gem "pry-byebug" end end end diff --git a/docker-compose.yml b/docker-compose.yml index f2ec382c..c8e754b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: httpx: environment: - HTTPBIN_HOST=nghttp2 + - HTTPBIN_NO_PROXY_HOST=http://httpbin:8000 - HTTPX_HTTP_PROXY=http://proxyuser:password@httpproxy:3128 - HTTPX_HTTPS_PROXY=http://proxyuser:password@httpproxy:3128 - HTTPX_HTTP2_PROXY=http://proxyuser:password@http2proxy:80 diff --git a/lib/httpx/plugins/proxy.rb b/lib/httpx/plugins/proxy.rb index 8fb7f977..902f5b36 100644 --- a/lib/httpx/plugins/proxy.rb +++ b/lib/httpx/plugins/proxy.rb @@ -105,6 +105,10 @@ module HTTPX end return if @_proxy_uris.empty? + proxy = options.proxy + + return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty? + proxy_opts = { uri: @_proxy_uris.first } proxy_opts = options.proxy.merge(proxy_opts) if options.proxy proxy_opts @@ -117,7 +121,9 @@ module HTTPX next_proxy = proxy_uris(uri, options) raise Error, "Failed to connect to proxy" unless next_proxy - proxy_options = options.merge(proxy: Parameters.new(**next_proxy)) + proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host + + proxy_options = options.merge(proxy: proxy) connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options) unless connections.nil? || connections.include?(connection) connections << connection diff --git a/sig/plugins/proxy.rbs b/sig/plugins/proxy.rbs index 41dd8238..9bfd41fa 100644 --- a/sig/plugins/proxy.rbs +++ b/sig/plugins/proxy.rbs @@ -21,7 +21,7 @@ module HTTPX private - def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **extra) -> untyped + def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **untyped) -> untyped end def self.configure: (singleton(Session)) -> void diff --git a/test/support/http_helpers.rb b/test/support/http_helpers.rb index d6589d0d..fe1e878e 100644 --- a/test/support/http_helpers.rb +++ b/test/support/http_helpers.rb @@ -21,4 +21,8 @@ module HTTPHelpers def httpbin ENV.fetch("HTTPBIN_HOST", "nghttp2.org/httpbin") end + + def httpbin_no_proxy + URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "httpbin.org")) + end end diff --git a/test/support/proxy_helper.rb b/test/support/proxy_helper.rb index 029e1922..cd2aa556 100644 --- a/test/support/proxy_helper.rb +++ b/test/support/proxy_helper.rb @@ -79,7 +79,7 @@ module ProxyHelper id = node.attribute("id") next unless id - id.value == "proxylisttable" + id.value == "list" end row ? row.css("tr") : [] end diff --git a/test/support/proxy_response_detector.rb b/test/support/proxy_response_detector.rb new file mode 100644 index 00000000..e9eabd4b --- /dev/null +++ b/test/support/proxy_response_detector.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module ProxyResponseDetector + module RequestMethods + attr_writer :proxied + + def proxied? + @proxied + end + end + + module ResponseMethods + def proxied? + @request.proxied? + end + end + + module ConnectionMethods + def send(request) + return super unless @options.respond_to?(:proxy) && @options.proxy + + proxy_uri = URI(@options.proxy.uri) + + request.proxied = @origin.host == proxy_uri.host && @origin.port == proxy_uri.port + + super + end + end +end diff --git a/test/support/requests/plugins/proxy.rb b/test/support/requests/plugins/proxy.rb index 912ea514..a28ebdae 100644 --- a/test/support/requests/plugins/proxy.rb +++ b/test/support/requests/plugins/proxy.rb @@ -16,33 +16,56 @@ module Requests assert_raises(HTTPX::HTTPProxyError) { session.get(uri) } end - def test_plugin_http_http1_proxy + def test_plugin_http_http_proxy return unless origin.start_with?("http://") - session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").with_proxy(uri: http_proxy) + session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").plugin(ProxyResponseDetector).with_proxy(uri: http_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? + end + + def test_plugin_http_no_proxy + return unless origin.start_with?("http://") + + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy, no_proxy: [httpbin_no_proxy.host]) + + # proxy + uri = build_uri("/get") + response = session.get(uri) + verify_status(response, 200) + verify_body_length(response) + assert response.proxied? + + # no proxy + no_proxy_uri = build_uri("/get", httpbin_no_proxy) + no_proxy_response = session.get(no_proxy_uri) + verify_status(no_proxy_response, 200) + verify_body_length(no_proxy_response) + assert !no_proxy_response.proxied? end def test_plugin_http_h2_proxy return unless origin.start_with?("http://") - session = HTTPX.plugin(:proxy, fallback_protocol: "h2").with_proxy(uri: http2_proxy) + session = HTTPX.plugin(:proxy, fallback_protocol: "h2").plugin(ProxyResponseDetector).with_proxy(uri: http2_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_https_connect_http1_proxy # return unless origin.start_with?("https://") - session = HTTPX.plugin(:proxy).with_proxy(uri: http_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end if OpenSSL::SSL::SSLContext.method_defined?(:alpn_protocols=) # TODO: uncomment when supporting H2 CONNECT @@ -59,11 +82,13 @@ module Requests def test_plugin_http_next_proxy session = HTTPX.plugin(SessionWithPool) .plugin(:proxy) + .plugin(ProxyResponseDetector) .with_proxy(uri: ["http://unavailable-proxy", *http_proxy]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_http_proxy_auth_options @@ -75,7 +100,7 @@ module Requests auth_proxy.user = nil auth_proxy.password = nil - session = HTTPX.plugin(:proxy).with_proxy( + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy( uri: auth_proxy.to_s, username: user, password: pass @@ -84,6 +109,7 @@ module Requests response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_http_proxy_auth_error @@ -109,6 +135,7 @@ module Requests auth_proxy.password = nil session = HTTPX.plugin(:proxy) + .plugin(ProxyResponseDetector) .with_proxy_digest_auth( uri: auth_proxy.to_s, username: user, @@ -118,25 +145,28 @@ module Requests response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks4_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy_ip proxy = URI(socks4_proxy.first) proxy.host = Resolv.getaddress(proxy.host) - session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy]) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: [proxy]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy_error @@ -150,23 +180,25 @@ module Requests end def test_plugin_socks4a_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks4a_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4a_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks5_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks5_ipv4_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy) uri = URI(build_uri("/get")) hostname = uri.host @@ -176,6 +208,7 @@ module Requests response = session.get(uri, headers: { "host" => uri.authority }, ssl: { hostname: hostname }) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end # TODO: enable when docker-compose supports ipv6 out of the box @@ -213,11 +246,12 @@ module Requests end def test_plugin_ssh_proxy - session = HTTPX.plugin(:"proxy/ssh").with_proxy(uri: ssh_proxy, - username: "root", - auth_methods: %w[publickey], - host_key: "ssh-rsa", - keys: %w[test/support/ssh/ssh_host_ed25519_key]) + session = HTTPX.plugin(:"proxy/ssh") + .with_proxy(uri: ssh_proxy, + username: "root", + auth_methods: %w[publickey], + host_key: "ssh-rsa", + keys: %w[test/support/ssh/ssh_host_ed25519_key]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200)