From 7c1ed567142da379c32f420966a2a7afe4d6b074 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Thu, 4 Aug 2022 14:01:47 +0100 Subject: [PATCH 1/3] fixing local proxy list --- test/support/proxy_helper.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/support/proxy_helper.rb b/test/support/proxy_helper.rb index 029e1922..cd2aa556 100644 --- a/test/support/proxy_helper.rb +++ b/test/support/proxy_helper.rb @@ -79,7 +79,7 @@ module ProxyHelper id = node.attribute("id") next unless id - id.value == "proxylisttable" + id.value == "list" end row ? row.css("tr") : [] end From 39beff84abc2c998f7a27881479cb4fe7297eea5 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Thu, 4 Aug 2022 14:02:44 +0100 Subject: [PATCH 2/3] added ability to check if request has been proxied --- test/support/proxy_response_detector.rb | 29 +++++++++++++++ test/support/requests/plugins/proxy.rb | 47 +++++++++++++++++-------- 2 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 test/support/proxy_response_detector.rb diff --git a/test/support/proxy_response_detector.rb b/test/support/proxy_response_detector.rb new file mode 100644 index 00000000..3a084908 --- /dev/null +++ b/test/support/proxy_response_detector.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module ProxyResponseDetector + module RequestMethods + attr_writer :proxied + + def proxied? + @proxied + end + end + + module ResponseMethods + def proxied? + @request.proxied? + end + end + + module ConnectionMethods + def send(request) + return super unless @options.respond_to?(:proxy) + + proxy_uri = URI(@options.proxy.uri) + + request.proxied = @origin.host == proxy_uri.host && @origin.port == proxy_uri.port + + super + end + end +end diff --git a/test/support/requests/plugins/proxy.rb b/test/support/requests/plugins/proxy.rb index 912ea514..f40c1f1d 100644 --- a/test/support/requests/plugins/proxy.rb +++ b/test/support/requests/plugins/proxy.rb @@ -19,30 +19,33 @@ module Requests def test_plugin_http_http1_proxy return unless origin.start_with?("http://") - session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").with_proxy(uri: http_proxy) + session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").plugin(ProxyResponseDetector).with_proxy(uri: http_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_http_h2_proxy return unless origin.start_with?("http://") - session = HTTPX.plugin(:proxy, fallback_protocol: "h2").with_proxy(uri: http2_proxy) + session = HTTPX.plugin(:proxy, fallback_protocol: "h2").plugin(ProxyResponseDetector).with_proxy(uri: http2_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_https_connect_http1_proxy # return unless origin.start_with?("https://") - session = HTTPX.plugin(:proxy).with_proxy(uri: http_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end if OpenSSL::SSL::SSLContext.method_defined?(:alpn_protocols=) # TODO: uncomment when supporting H2 CONNECT @@ -59,11 +62,13 @@ module Requests def test_plugin_http_next_proxy session = HTTPX.plugin(SessionWithPool) .plugin(:proxy) + .plugin(ProxyResponseDetector) .with_proxy(uri: ["http://unavailable-proxy", *http_proxy]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_http_proxy_auth_options @@ -75,7 +80,7 @@ module Requests auth_proxy.user = nil auth_proxy.password = nil - session = HTTPX.plugin(:proxy).with_proxy( + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy( uri: auth_proxy.to_s, username: user, password: pass @@ -84,6 +89,7 @@ module Requests response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_http_proxy_auth_error @@ -93,10 +99,11 @@ module Requests no_auth_proxy.user = nil no_auth_proxy.password = nil - session = HTTPX.plugin(:proxy).with_proxy(uri: no_auth_proxy.to_s) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: no_auth_proxy.to_s) uri = build_uri("/get") response = session.get(uri) verify_status(response, 407) + assert response.proxied? end def test_plugin_http_proxy_digest_auth @@ -109,6 +116,7 @@ module Requests auth_proxy.password = nil session = HTTPX.plugin(:proxy) + .plugin(ProxyResponseDetector) .with_proxy_digest_auth( uri: auth_proxy.to_s, username: user, @@ -118,25 +126,28 @@ module Requests response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks4_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy_ip proxy = URI(socks4_proxy.first) proxy.host = Resolv.getaddress(proxy.host) - session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy]) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: [proxy]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks4_proxy_error @@ -150,23 +161,25 @@ module Requests end def test_plugin_socks4a_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks4a_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4a_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks5_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end def test_plugin_socks5_ipv4_proxy - session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy) + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy) uri = URI(build_uri("/get")) hostname = uri.host @@ -176,6 +189,7 @@ module Requests response = session.get(uri, headers: { "host" => uri.authority }, ssl: { hostname: hostname }) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end # TODO: enable when docker-compose supports ipv6 out of the box @@ -213,15 +227,18 @@ module Requests end def test_plugin_ssh_proxy - session = HTTPX.plugin(:"proxy/ssh").with_proxy(uri: ssh_proxy, - username: "root", - auth_methods: %w[publickey], - host_key: "ssh-rsa", - keys: %w[test/support/ssh/ssh_host_ed25519_key]) + session = HTTPX.plugin(:"proxy/ssh") + .plugin(ProxyResponseDetector) + .with_proxy(uri: ssh_proxy, + username: "root", + auth_methods: %w[publickey], + host_key: "ssh-rsa", + keys: %w[test/support/ssh/ssh_host_ed25519_key]) uri = build_uri("/get") response = session.get(uri) verify_status(response, 200) verify_body_length(response) + assert response.proxied? end if ENV.key?("HTTPX_SSH_PROXY") && RUBY_ENGINE != "jruby" end end From 43016795f39a60cae8d5adec688b9593baa84b36 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Thu, 4 Aug 2022 22:42:57 +0100 Subject: [PATCH 3/3] introducing the :no_proxy option can be passed in the `:proxy` option hash, and receives domains, as strings, which requests should not go through the proxy. --- Gemfile | 8 ++++++-- docker-compose.yml | 1 + lib/httpx/plugins/proxy.rb | 8 +++++++- sig/plugins/proxy.rbs | 2 +- test/support/http_helpers.rb | 4 ++++ test/support/proxy_response_detector.rb | 2 +- test/support/requests/plugins/proxy.rb | 27 ++++++++++++++++++++----- 7 files changed, 42 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index 20896089..5d30c31c 100644 --- a/Gemfile +++ b/Gemfile @@ -18,7 +18,11 @@ group :test do gem "ruby-ntlm" gem "sentry-ruby" if RUBY_VERSION >= "2.4" gem "spy" - gem "webmock" + if RUBY_VERSION < "2.3.0" + gem "webmock", "< 3.15.0" + else + gem "webmock" + end gem "websocket-driver" gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2" @@ -115,8 +119,8 @@ group :assorted do if RUBY_VERSION < "2.2" gem "pry-byebug", "~> 3.4.3" else - gem "pry-byebug" gem "debug" if RUBY_VERSION >= "3.1.0" + gem "pry-byebug" end end end diff --git a/docker-compose.yml b/docker-compose.yml index f2ec382c..c8e754b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,7 @@ services: httpx: environment: - HTTPBIN_HOST=nghttp2 + - HTTPBIN_NO_PROXY_HOST=http://httpbin:8000 - HTTPX_HTTP_PROXY=http://proxyuser:password@httpproxy:3128 - HTTPX_HTTPS_PROXY=http://proxyuser:password@httpproxy:3128 - HTTPX_HTTP2_PROXY=http://proxyuser:password@http2proxy:80 diff --git a/lib/httpx/plugins/proxy.rb b/lib/httpx/plugins/proxy.rb index 8fb7f977..902f5b36 100644 --- a/lib/httpx/plugins/proxy.rb +++ b/lib/httpx/plugins/proxy.rb @@ -105,6 +105,10 @@ module HTTPX end return if @_proxy_uris.empty? + proxy = options.proxy + + return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty? + proxy_opts = { uri: @_proxy_uris.first } proxy_opts = options.proxy.merge(proxy_opts) if options.proxy proxy_opts @@ -117,7 +121,9 @@ module HTTPX next_proxy = proxy_uris(uri, options) raise Error, "Failed to connect to proxy" unless next_proxy - proxy_options = options.merge(proxy: Parameters.new(**next_proxy)) + proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host + + proxy_options = options.merge(proxy: proxy) connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options) unless connections.nil? || connections.include?(connection) connections << connection diff --git a/sig/plugins/proxy.rbs b/sig/plugins/proxy.rbs index 41dd8238..9bfd41fa 100644 --- a/sig/plugins/proxy.rbs +++ b/sig/plugins/proxy.rbs @@ -21,7 +21,7 @@ module HTTPX private - def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **extra) -> untyped + def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **untyped) -> untyped end def self.configure: (singleton(Session)) -> void diff --git a/test/support/http_helpers.rb b/test/support/http_helpers.rb index d6589d0d..fe1e878e 100644 --- a/test/support/http_helpers.rb +++ b/test/support/http_helpers.rb @@ -21,4 +21,8 @@ module HTTPHelpers def httpbin ENV.fetch("HTTPBIN_HOST", "nghttp2.org/httpbin") end + + def httpbin_no_proxy + URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "httpbin.org")) + end end diff --git a/test/support/proxy_response_detector.rb b/test/support/proxy_response_detector.rb index 3a084908..e9eabd4b 100644 --- a/test/support/proxy_response_detector.rb +++ b/test/support/proxy_response_detector.rb @@ -17,7 +17,7 @@ module ProxyResponseDetector module ConnectionMethods def send(request) - return super unless @options.respond_to?(:proxy) + return super unless @options.respond_to?(:proxy) && @options.proxy proxy_uri = URI(@options.proxy.uri) diff --git a/test/support/requests/plugins/proxy.rb b/test/support/requests/plugins/proxy.rb index f40c1f1d..a28ebdae 100644 --- a/test/support/requests/plugins/proxy.rb +++ b/test/support/requests/plugins/proxy.rb @@ -16,7 +16,7 @@ module Requests assert_raises(HTTPX::HTTPProxyError) { session.get(uri) } end - def test_plugin_http_http1_proxy + def test_plugin_http_http_proxy return unless origin.start_with?("http://") session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").plugin(ProxyResponseDetector).with_proxy(uri: http_proxy) @@ -27,6 +27,26 @@ module Requests assert response.proxied? end + def test_plugin_http_no_proxy + return unless origin.start_with?("http://") + + session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy, no_proxy: [httpbin_no_proxy.host]) + + # proxy + uri = build_uri("/get") + response = session.get(uri) + verify_status(response, 200) + verify_body_length(response) + assert response.proxied? + + # no proxy + no_proxy_uri = build_uri("/get", httpbin_no_proxy) + no_proxy_response = session.get(no_proxy_uri) + verify_status(no_proxy_response, 200) + verify_body_length(no_proxy_response) + assert !no_proxy_response.proxied? + end + def test_plugin_http_h2_proxy return unless origin.start_with?("http://") @@ -99,11 +119,10 @@ module Requests no_auth_proxy.user = nil no_auth_proxy.password = nil - session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: no_auth_proxy.to_s) + session = HTTPX.plugin(:proxy).with_proxy(uri: no_auth_proxy.to_s) uri = build_uri("/get") response = session.get(uri) verify_status(response, 407) - assert response.proxied? end def test_plugin_http_proxy_digest_auth @@ -228,7 +247,6 @@ module Requests def test_plugin_ssh_proxy session = HTTPX.plugin(:"proxy/ssh") - .plugin(ProxyResponseDetector) .with_proxy(uri: ssh_proxy, username: "root", auth_methods: %w[publickey], @@ -238,7 +256,6 @@ module Requests response = session.get(uri) verify_status(response, 200) verify_body_length(response) - assert response.proxied? end if ENV.key?("HTTPX_SSH_PROXY") && RUBY_ENGINE != "jruby" end end