Merge branch 'issue-202' into 'master'

introducing the :no_proxy option

Closes #202

See merge request honeyryderchuck/httpx!217

Gemfile

@@ -18,7 +18,11 @@ group :test do
   gem "ruby-ntlm"
   gem "sentry-ruby" if RUBY_VERSION >= "2.4"
   gem "spy"
-  gem "webmock"
+  if RUBY_VERSION < "2.3.0"
+    gem "webmock", "< 3.15.0"
+  else
+    gem "webmock"
+  end
   gem "websocket-driver"
 
   gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2"
@@ -115,8 +119,8 @@ group :assorted do
     if RUBY_VERSION < "2.2"
       gem "pry-byebug", "~> 3.4.3"
     else
-      gem "pry-byebug"
       gem "debug" if RUBY_VERSION >= "3.1.0"
+      gem "pry-byebug"
     end
   end
 end

docker-compose.yml

@@ -3,6 +3,7 @@ services:
   httpx:
     environment:
       - HTTPBIN_HOST=nghttp2
+      - HTTPBIN_NO_PROXY_HOST=http://httpbin:8000
       - HTTPX_HTTP_PROXY=http://proxyuser:password@httpproxy:3128
       - HTTPX_HTTPS_PROXY=http://proxyuser:password@httpproxy:3128
       - HTTPX_HTTP2_PROXY=http://proxyuser:password@http2proxy:80

lib/httpx/plugins/proxy.rb

@@ -105,6 +105,10 @@ module HTTPX
           end
           return if @_proxy_uris.empty?
 
+          proxy = options.proxy
+
+          return { uri: uri.host } if proxy && proxy.key?(:no_proxy) && !Array(proxy[:no_proxy]).grep(uri.host).empty?
+
           proxy_opts = { uri: @_proxy_uris.first }
           proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
           proxy_opts
@@ -117,7 +121,9 @@ module HTTPX
           next_proxy = proxy_uris(uri, options)
           raise Error, "Failed to connect to proxy" unless next_proxy
 
-          proxy_options = options.merge(proxy: Parameters.new(**next_proxy))
+          proxy = Parameters.new(**next_proxy) unless next_proxy[:uri] == uri.host
+
+          proxy_options = options.merge(proxy: proxy)
           connection = pool.find_connection(uri, proxy_options) || build_connection(uri, proxy_options)
           unless connections.nil? || connections.include?(connection)
             connections << connection

sig/plugins/proxy.rbs

@@ -21,7 +21,7 @@ module HTTPX
 
         private
 
-        def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **extra) -> untyped
+        def initialize: (uri: generic_uri, ?scheme: String, ?username: String, ?password: String, **untyped) -> untyped
       end
 
       def self.configure: (singleton(Session)) -> void

test/support/http_helpers.rb

@@ -21,4 +21,8 @@ module HTTPHelpers
   def httpbin
     ENV.fetch("HTTPBIN_HOST", "nghttp2.org/httpbin")
   end
+
+  def httpbin_no_proxy
+    URI(ENV.fetch("HTTPBIN_NO_PROXY_HOST", "httpbin.org"))
+  end
 end

test/support/proxy_helper.rb

@@ -79,7 +79,7 @@ module ProxyHelper
       id = node.attribute("id")
       next unless id
 
-      id.value == "proxylisttable"
+      id.value == "list"
     end
     row ? row.css("tr") : []
   end

test/support/proxy_response_detector.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module ProxyResponseDetector
+  module RequestMethods
+    attr_writer :proxied
+
+    def proxied?
+      @proxied
+    end
+  end
+
+  module ResponseMethods
+    def proxied?
+      @request.proxied?
+    end
+  end
+
+  module ConnectionMethods
+    def send(request)
+      return super unless @options.respond_to?(:proxy) && @options.proxy
+
+      proxy_uri = URI(@options.proxy.uri)
+
+      request.proxied = @origin.host == proxy_uri.host && @origin.port == proxy_uri.port
+
+      super
+    end
+  end
+end

test/support/requests/plugins/proxy.rb

@@ -16,33 +16,56 @@ module Requests
         assert_raises(HTTPX::HTTPProxyError) { session.get(uri) }
       end
 
-      def test_plugin_http_http1_proxy
+      def test_plugin_http_http_proxy
         return unless origin.start_with?("http://")
 
-        session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").with_proxy(uri: http_proxy)
+        session = HTTPX.plugin(:proxy, fallback_protocol: "http/1.1").plugin(ProxyResponseDetector).with_proxy(uri: http_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
+      end
+
+      def test_plugin_http_no_proxy
+        return unless origin.start_with?("http://")
+
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy, no_proxy: [httpbin_no_proxy.host])
+
+        # proxy
+        uri = build_uri("/get")
+        response = session.get(uri)
+        verify_status(response, 200)
+        verify_body_length(response)
+        assert response.proxied?
+
+        # no proxy
+        no_proxy_uri = build_uri("/get", httpbin_no_proxy)
+        no_proxy_response = session.get(no_proxy_uri)
+        verify_status(no_proxy_response, 200)
+        verify_body_length(no_proxy_response)
+        assert !no_proxy_response.proxied?
       end
 
       def test_plugin_http_h2_proxy
         return unless origin.start_with?("http://")
 
-        session = HTTPX.plugin(:proxy, fallback_protocol: "h2").with_proxy(uri: http2_proxy)
+        session = HTTPX.plugin(:proxy, fallback_protocol: "h2").plugin(ProxyResponseDetector).with_proxy(uri: http2_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_https_connect_http1_proxy
         # return unless origin.start_with?("https://")
-        session = HTTPX.plugin(:proxy).with_proxy(uri: http_proxy)
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: http_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end if OpenSSL::SSL::SSLContext.method_defined?(:alpn_protocols=)
 
       # TODO: uncomment when supporting H2 CONNECT
@@ -59,11 +82,13 @@ module Requests
       def test_plugin_http_next_proxy
         session = HTTPX.plugin(SessionWithPool)
                        .plugin(:proxy)
+                       .plugin(ProxyResponseDetector)
                        .with_proxy(uri: ["http://unavailable-proxy", *http_proxy])
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_http_proxy_auth_options
@@ -75,7 +100,7 @@ module Requests
         auth_proxy.user = nil
         auth_proxy.password = nil
 
-        session = HTTPX.plugin(:proxy).with_proxy(
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(
           uri: auth_proxy.to_s,
           username: user,
           password: pass
@@ -84,6 +109,7 @@ module Requests
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_http_proxy_auth_error
@@ -109,6 +135,7 @@ module Requests
         auth_proxy.password = nil
 
         session = HTTPX.plugin(:proxy)
+                       .plugin(ProxyResponseDetector)
                        .with_proxy_digest_auth(
                          uri: auth_proxy.to_s,
                          username: user,
@@ -118,25 +145,28 @@ module Requests
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_socks4_proxy
-        session = HTTPX.plugin(:proxy).with_proxy(uri: socks4_proxy)
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_socks4_proxy_ip
         proxy = URI(socks4_proxy.first)
         proxy.host = Resolv.getaddress(proxy.host)
 
-        session = HTTPX.plugin(:proxy).with_proxy(uri: [proxy])
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: [proxy])
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_socks4_proxy_error
@@ -150,23 +180,25 @@ module Requests
       end
 
       def test_plugin_socks4a_proxy
-        session = HTTPX.plugin(:proxy).with_proxy(uri: socks4a_proxy)
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks4a_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_socks5_proxy
-        session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy)
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy)
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       def test_plugin_socks5_ipv4_proxy
-        session = HTTPX.plugin(:proxy).with_proxy(uri: socks5_proxy)
+        session = HTTPX.plugin(:proxy).plugin(ProxyResponseDetector).with_proxy(uri: socks5_proxy)
         uri = URI(build_uri("/get"))
         hostname = uri.host
 
@@ -176,6 +208,7 @@ module Requests
         response = session.get(uri, headers: { "host" => uri.authority }, ssl: { hostname: hostname })
         verify_status(response, 200)
         verify_body_length(response)
+        assert response.proxied?
       end
 
       # TODO: enable when docker-compose supports ipv6 out of the box
@@ -213,11 +246,12 @@ module Requests
       end
 
       def test_plugin_ssh_proxy
-        session = HTTPX.plugin(:"proxy/ssh").with_proxy(uri: ssh_proxy,
-                                                        username: "root",
-                                                        auth_methods: %w[publickey],
-                                                        host_key: "ssh-rsa",
-                                                        keys: %w[test/support/ssh/ssh_host_ed25519_key])
+        session = HTTPX.plugin(:"proxy/ssh")
+                       .with_proxy(uri: ssh_proxy,
+                                   username: "root",
+                                   auth_methods: %w[publickey],
+                                   host_key: "ssh-rsa",
+                                   keys: %w[test/support/ssh/ssh_host_ed25519_key])
         uri = build_uri("/get")
         response = session.get(uri)
         verify_status(response, 200)