bumped version to 0.21.0

webdav

Closes #201

See merge request honeyryderchuck/httpx!219

Gemfile

@@ -23,6 +23,7 @@ group :test do
   else
     gem "webmock"
   end
+  gem "nokogiri"
   gem "websocket-driver"
 
   gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2.0"

doc/release_notes/0_21_0.md

@@ -0,0 +1,94 @@
+# 0.21.0
+
+## Features
+
+### `:write_timeout`, `:read_timeout` and `:request_timeout`
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Timeouts
+
+The following timeouts are now supported:
+
+* `:write_timeout`: total time (in seconds) to write a request to the server;
+* `:read_timeout`: total time (in seconds) to read aa response from the server;
+* `:request_timeout`: tracks both of the above (time to write the request and read a response);
+
+```ruby
+HTTPX.with(timeout: { request_timeout: 60}).get(...
+```
+
+Just like `:connect_timeout`, the new timeouts are deadline-oriented, rather than op-oriented, meaning that they do not reset on each socket operation (as most ruby HTTP clients do).
+
+None of them has a default value, in order not to break integrations, but that'll change in a future v1, where they'll become the default timeouts.
+
+### Circuit Breaker plugin
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/Circuit-Breaker
+
+The `:circuit_breaker` plugin wraps around errors happening when performing HTTP requests, and support options for setting maximum number of attempts before circuit opens (`:circuit_breaker_max_attempts`), period after which attempts should be reset (`:circuit_breaker_reset_attempts_in`), timespan until circuit half-opens (`circuit_breaker_break_in`), respective half-open drip rate (`:circuit_breaker_half_open_drip_rate`), and a callback to do your own check on whether a response has failed, in case you want HTTP level errors to be marked as failed attempts (`:circuit_breaker_break_on`).
+
+```ruby
+http = HTTPX.plugin(:circuit_breaker)
+# that's it!
+http.get(...
+```
+
+### WebDAV plugin
+
+https://gitlab.com/honeyryderchuck/httpx/-/wikis/WebDav
+
+The `:webdav` introduces some "convenience" methods to perform common WebDAV operations.
+
+```ruby
+webdav = HTTPX.plugin(:webdav, origin: "http://webdav-server")
+              .plugin(:digest_authentication).digest_auth("user", "pass")
+
+res = webdav.put("/file.html", body: "this is the file body")
+res = webdav.copy("/file.html", "/newdir/copy.html")
+# ...
+```
+
+### XML transcoder, `:xml` option and `response.xml`
+
+A new transcoder was added fot the XML mime type, which requires `"nokogiri"` to be installed; it can both serialize Nokogiri nodes in a request, and parse response content into nokogiri nodes:
+
+```ruby
+response = HTTPX.post("https://xml-server.com", xml: Nokogiri::XML("<xml ..."))
+response.xml #=> #(Document:0x16e4 { name = "document", children = ...
+```
+
+## Improvements
+
+### `:proxy` plugin: `:no_proxy` option
+
+Support was added, in the `:proxy` plugin, to declare domains, either via regexp patterns, or strings, for which requests should bypass the proxy.
+
+```ruby
+http = HTTPX.plugin(:proxy).with_proxy(
+    uri: "http://10.10.0.1:51432",
+    no_proxy: ["gitlab.local", /*.google.com/]
+)
+http.get("https://duckduckgo.com/?q=httpx") #=> proxied
+http.get("https://google.com/?q=httpx") #=> not proxied
+http.get("https://gitlab.com") #=> proxied
+http.get("https://gitlab.local") #=> not proxied
+```
+
+### OOTB support for other JSON libraries
+
+If one of `multi_json`, `oj` or `yajl` is available, all `httpx` operations doing JSON parsing or dumping will use it (the `json` standard library will be used otherwise).
+
+```ruby
+require "oj"
+require "httpx"
+
+response = HTTPX.post("https://somedomain.json", json: { "foo" => "bar" }) # will use "oj"
+puts response.json # will use "oj"
+```
+
+## Bugfixes
+
+* `:expect` plugin: `:expect_timeout` can accept floats (not just integers).
+
+## Chore
+
+* DoH `:https` resolver: support was removed for the "application/dns-json" mime-type (it was only supported in practice by the Google DoH resolver, which has since added support for the standardized "application/dns-message").

docker-compose.yml

@@ -24,6 +24,7 @@ services:
       - AWS_ACCESS_KEY_ID=test
       - AWS_SECRET_ACCESS_KEY=test
       - AMZ_HOST=aws:4566
+      - WEBDAV_HOST=webdav
     image: ruby:alpine
     privileged: true
     depends_on:
@@ -34,6 +35,7 @@ services:
       - nghttp2
       - aws
       - ws-echo-server
+      - webdav
     volumes:
       - ./:/home
     links:
@@ -137,3 +139,10 @@ services:
     ports:
       - 8083:80
     image: jmalloc/echo-server
+
+  webdav:
+    image: bytemark/webdav
+    environment:
+      - AUTH_TYPE=Basic
+      - USERNAME=user
+      - PASSWORD=pass

lib/httpx/extensions.rb

@@ -143,10 +143,6 @@ module HTTPX
   end
 
   module RegexpExtensions
-    # If you wonder why this is there: the oauth feature uses a refinement to enhance the
-    # Regexp class locally with #match? , but this is never tested, because ActiveSupport
-    # monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
-    # :nocov:
     refine(Regexp) do
       def match?(*args)
         !match(*args).nil?

lib/httpx/options.rb

@@ -201,7 +201,7 @@ module HTTPX
     end
 
     %i[
-      params form json body ssl http2_settings
+      params form json xml body ssl http2_settings
       request_class response_class headers_class request_body_class
       response_body_class connection_class options_class
       io fallback_protocol debug debug_level transport_options resolver_class resolver_options
@@ -210,7 +210,7 @@ module HTTPX
       def_option(method_name)
     end
 
-    REQUEST_IVARS = %i[@params @form @json @body].freeze
+    REQUEST_IVARS = %i[@params @form @xml @json @body].freeze
     private_constant :REQUEST_IVARS
 
     def ==(other)

lib/httpx/plugins/webdav.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    #
+    # This plugin implements convenience methods for performing WEBDAV requests.
+    #
+    # https://gitlab.com/honeyryderchuck/httpx/wikis/WEBDAV
+    #
+    module WebDav
+      module InstanceMethods
+        def copy(src, dest)
+          request(:copy, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def move(src, dest)
+          request(:move, src, headers: { "destination" => @options.origin.merge(dest) })
+        end
+
+        def lock(path, timeout: nil, &blk)
+          headers = {}
+          headers["timeout"] = if timeout && timeout.positive?
+            "Second-#{timeout}"
+          else
+            "Infinite, Second-4100000000"
+          end
+          xml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" \
+                "<D:lockinfo xmlns:D=\"DAV:\">" \
+                "<D:lockscope><D:exclusive/></D:lockscope>" \
+                "<D:locktype><D:write/></D:locktype>" \
+                "<D:owner>null</D:owner>" \
+                "</D:lockinfo>"
+          response = request(:lock, path, headers: headers, xml: xml)
+
+          return response unless blk && response.status == 200
+
+          lock_token = response.headers["lock-token"]
+
+          begin
+            blk.call(response)
+          ensure
+            unlock(path, lock_token)
+          end
+        end
+
+        def unlock(path, lock_token)
+          request(:unlock, path, headers: { "lock-token" => lock_token })
+        end
+
+        def mkcol(dir)
+          request(:mkcol, dir)
+        end
+
+        def propfind(path, xml = nil)
+          body = case xml
+                 when :acl
+                   '<?xml version="1.0" encoding="utf-8" ?><D:propfind xmlns:D="DAV:"><D:prop><D:owner/>' \
+                   "<D:supported-privilege-set/><D:current-user-privilege-set/><D:acl/></D:prop></D:propfind>"
+                 when nil
+                   '<?xml version="1.0" encoding="utf-8"?><DAV:propfind xmlns:DAV="DAV:"><DAV:allprop/></DAV:propfind>'
+                 else
+                   xml
+          end
+
+          request(:propfind, path, headers: { "depth" => "1" }, xml: body)
+        end
+
+        def proppatch(path, xml)
+          body = "<?xml version=\"1.0\"?>" \
+                 "<D:propertyupdate xmlns:D=\"DAV:\" xmlns:Z=\"http://ns.example.com/standards/z39.50/\">#{xml}</D:propertyupdate>"
+          request(:proppatch, path, xml: body)
+        end
+        # %i[ orderpatch acl report search]
+      end
+    end
+    register_plugin(:webdav, WebDav)
+  end
+end

lib/httpx/request.rb

@@ -9,29 +9,6 @@ module HTTPX
     include Callbacks
     using URIExtensions
 
-    METHODS = [
-      # RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1
-      :options, :get, :head, :post, :put, :delete, :trace, :connect,
-
-      # RFC 2518: HTTP Extensions for Distributed Authoring -- WEBDAV
-      :propfind, :proppatch, :mkcol, :copy, :move, :lock, :unlock,
-
-      # RFC 3648: WebDAV Ordered Collections Protocol
-      :orderpatch,
-
-      # RFC 3744: WebDAV Access Control Protocol
-      :acl,
-
-      # RFC 6352: vCard Extensions to WebDAV -- CardDAV
-      :report,
-
-      # RFC 5789: PATCH Method for HTTP
-      :patch,
-
-      # draft-reschke-webdav-search: WebDAV Search
-      :search
-    ].freeze
-
     USER_AGENT = "httpx.rb/#{VERSION}"
 
     attr_reader :verb, :uri, :headers, :body, :state, :options, :response
@@ -54,8 +31,6 @@ module HTTPX
         @uri = origin.merge("#{base_path}#{@uri}")
       end
 
-      raise(Error, "unknown method: #{verb}") unless METHODS.include?(@verb)
-
       @headers = @options.headers_class.new(@options.headers)
       @headers["user-agent"] ||= USER_AGENT
       @headers["accept"]     ||= "*/*"
@@ -187,6 +162,8 @@ module HTTPX
           Transcoder.registry("form").encode(options.form)
         elsif options.json
           Transcoder.registry("json").encode(options.json)
+        elsif options.xml
+          Transcoder.registry("xml").encode(options.xml)
         end
         return if @body.nil?
 

lib/httpx/response.rb

@@ -94,6 +94,10 @@ module HTTPX
       decode("form")
     end
 
+    def xml
+      decode("xml")
+    end
+
     private
 
     def decode(format, *args)

lib/httpx/transcoder.rb

@@ -90,4 +90,5 @@ end
 require "httpx/transcoder/body"
 require "httpx/transcoder/form"
 require "httpx/transcoder/json"
+require "httpx/transcoder/xml"
 require "httpx/transcoder/chunker"

lib/httpx/transcoder/xml.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "forwardable"
+require "uri"
+
+module HTTPX::Transcoder
+  module Xml
+    using HTTPX::RegexpExtensions
+
+    module_function
+
+    MIME_TYPES = %r{\b(application|text)/(.+\+)?xml\b}.freeze
+
+    class Encoder
+      def initialize(xml)
+        @raw = xml
+      end
+
+      def content_type
+        charset = @raw.respond_to?(:encoding) ? @raw.encoding.to_s.downcase : "utf-8"
+        "application/xml; charset=#{charset}"
+      end
+
+      def bytesize
+        @raw.to_s.bytesize
+      end
+
+      def to_s
+        @raw.to_s
+      end
+    end
+
+    def encode(xml)
+      Encoder.new(xml)
+    end
+
+    begin
+      require "nokogiri"
+
+      # rubocop:disable Lint/DuplicateMethods
+      def decode(response)
+        content_type = response.content_type.mime_type
+
+        raise HTTPX::Error, "invalid form mime type (#{content_type})" unless MIME_TYPES.match?(content_type)
+
+        Nokogiri::XML.method(:parse)
+      end
+    rescue LoadError
+      def decode(_response)
+        raise HTTPX::Error, "\"nokogiri\" is required in order to decode XML"
+      end
+    end
+    # rubocop:enable Lint/DuplicateMethods
+  end
+  register "xml", Xml
+end

lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.20.5"
+  VERSION = "0.21.0"
 end

sig/transcoder/json.rbs

@@ -20,7 +20,7 @@ module HTTPX::Transcoder
 
       private
 
-      def initialize: (_ToJson json) -> untyped
+      def initialize: (_ToJson json) -> void
     end
   end
 end

sig/transcoder/xml.rbs

@@ -0,0 +1,21 @@
+module HTTPX::Transcoder
+  module XML
+
+    def self?.encode: (untyped xml) -> Encoder
+    def self?.decode: (HTTPX::Response response) -> _Decoder
+
+    class Encoder
+      @raw: untyped # can be nokogiri object
+
+      def content_type: () -> String
+
+      def bytesize: () -> (Integer | Float)
+
+      def to_s: () -> String
+
+      private
+
+      def initialize: (String xml) -> void
+    end
+  end
+end

test/http_test.rb

@@ -32,6 +32,7 @@ class HTTPTest < Minitest::Test
   include Plugins::GRPC if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.3.0"
   include Plugins::ResponseCache
   include Plugins::CircuitBreaker
+  include Plugins::WebDav
 
   def test_verbose_log
     log = StringIO.new

test/https_test.rb

@@ -33,6 +33,7 @@ class HTTPSTest < Minitest::Test
   include Plugins::GRPC if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.3.0"
   include Plugins::ResponseCache
   include Plugins::CircuitBreaker
+  include Plugins::WebDav
 
   def test_connection_coalescing
     coalesced_origin = "https://#{ENV["HTTPBIN_COALESCING_HOST"]}"

test/options_test.rb

@@ -45,7 +45,7 @@ class OptionsTest < Minitest::Test
     assert opt2.body == "fat", "body was not set"
   end
 
-  %i[form json].each do |meth|
+  %i[form json xml].each do |meth|
     define_method :"test_options_#{meth}" do
       opt1 = Options.new
       assert opt1.public_send(meth).nil?, "#{meth} shouldn't be set by default"
@@ -98,6 +98,7 @@ class OptionsTest < Minitest::Test
       :debug_level => 1,
       :params => nil,
       :json => nil,
+      :xml => nil,
       :body => nil,
       :window_size => 16_384,
       :body_threshold_size => 114_688,

test/request_test.rb

@@ -77,6 +77,13 @@ class RequestTest < Minitest::Test
     assert req.headers["content-length"] == "13", "content length is wrong"
   end
 
+  def test_request_body_xml
+    req = Request.new(:post, "http://example.com/", xml: "<xml></xml>")
+    assert !req.body.empty?, "body should exist"
+    assert req.headers["content-type"] == "application/xml; charset=utf-8", "content type is wrong"
+    assert req.headers["content-length"] == "11", "content length is wrong"
+  end
+
   private
 
   def resource

test/response_test.rb

@@ -156,6 +156,11 @@ class ResponseTest < Minitest::Test
     form_response << "богус"
     assert_raises(ArgumentError) { form_response.form }
 
+    xml_response = Response.new(request, 200, "2.0", { "content-type" => "application/xml; charset=utf-8" })
+    xml_response << "<xml></xml>"
+    xml = xml_response.xml
+    assert xml.is_a?(Nokogiri::XML::Node)
+
     form2_response = Response.new(request, 200, "2.0", { "content-type" => "application/x-www-form-urlencoded" })
     form2_response << "a[]=b&a[]=c&d[e]=f&g[h][i][j]=k&l[m][][n]=o&l[m][][p]=q&l[m][][n]=r&s[=t"
     assert form2_response.form == {

test/support/ci/build.sh

@@ -10,7 +10,7 @@ RUBY_ENGINE=`ruby -e 'puts RUBY_ENGINE'`
 IPTABLES=iptables-translate
 
 if [[ "$RUBY_ENGINE" = "truffleruby" ]]; then
-  dnf install -y iptables iproute which file idn2 git
+  dnf install -y iptables iproute which file idn2 git xz
 elif [[ "$RUBY_PLATFORM" = "java" ]]; then
   apt-get update && apt-get install -y build-essential iptables iproute2 file idn2 git
 elif [[ ${RUBY_VERSION:0:3} = "2.1" ]]; then

test/support/requests/plugins/webdav.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module Requests
+  module Plugins
+    module WebDav
+      def test_plugin_webdav_mkcol
+        # put file
+        webdav_client.delete("/mkcol_dir_test/")
+
+        response = webdav_client.mkcol("/mkcol_dir_test/")
+        verify_status(response, 201)
+      end
+
+      def test_plugin_webdav_copy
+        # put file
+        webdav_client.delete("/copied_copy.html")
+        webdav_client.put("/copy.html", body: "<html></html>")
+
+        response = webdav_client.get("/copied_copy.html")
+        verify_status(response, 404)
+        response = webdav_client.copy("/copy.html", "/copied_copy.html")
+        verify_status(response, 201)
+        response = webdav_client.get("/copied_copy.html")
+        verify_status(response, 200)
+        response = webdav_client.get("/copy.html")
+        verify_status(response, 200)
+      end
+
+      def test_plugin_webdav_move
+        # put file
+        webdav_client.delete("/moved_move.html")
+        webdav_client.put("/move.html", body: "<html></html>")
+
+        response = webdav_client.get("/moved_move.html")
+        verify_status(response, 404)
+        response = webdav_client.move("/move.html", "/moved_move.html")
+        verify_status(response, 201)
+        response = webdav_client.get("/move.html")
+        verify_status(response, 404)
+        response = webdav_client.get("/moved_move.html")
+        verify_status(response, 200)
+      end
+
+      def test_plugin_webdav_lock
+        # put file
+        webdav_client.put("/lockfile.html", body: "bang")
+        response = webdav_client.lock("/lockfile.html")
+        verify_status(response, 200)
+        lock_token = response.headers["lock-token"]
+
+        response = webdav_client.delete("/lockfile.html")
+        verify_status(response, 423)
+
+        response = webdav_client.unlock("/lockfile.html", lock_token)
+        verify_status(response, 204)
+
+        response = webdav_client.delete("/lockfile.html")
+        verify_status(response, 204)
+      end
+
+      def test_plugin_webdav_lock_blk
+        # put file
+        webdav_client.put("/lockfileblk.html", body: "bang")
+        webdav_client.lock("/lockfileblk.html") do |response|
+          verify_status(response, 200)
+
+          response = webdav_client.delete("/lockfileblk.html")
+          verify_status(response, 423)
+        end
+        response = webdav_client.delete("/lockfileblk.html")
+        verify_status(response, 204)
+      end
+
+      def test_plugin_webdav_propfind_proppatch
+        # put file
+        webdav_client.put("/propfind.html", body: "bang")
+        response = webdav_client.propfind("/propfind.html")
+        verify_status(response, 207)
+        xml = "<D:set>" \
+              "<D:prop>" \
+              "<Z:Authors>" \
+              "<Z:Author>Jim Bean</Z:Author>" \
+              "</Z:Authors>" \
+              "</D:prop>" \
+              "</D:set>"
+        response = webdav_client.proppatch("/propfind.html", xml)
+        verify_status(response, 207)
+
+        response = webdav_client.propfind("/propfind.html")
+        verify_status(response, 207)
+        assert response.to_s.include?("Jim Bean")
+      end
+
+      private
+
+      def webdav_client
+        @webdav_client ||= HTTPX.plugin(:basic_authentication).plugin(:webdav, origin: start_webdav_server).basic_auth("user", "pass")
+      end
+
+      def start_webdav_server
+        origin = ENV.fetch("WEBDAV_HOST")
+        "http://#{origin}"
+      end
+    end
+  end
+end

www/_data/versions.yml

@@ -1,4 +1,7 @@
 -
+  -
+    name: "0.21.0"
+    path: "0_21_0_md.html"
   -
     name: "0.20.5"
     path: "0_20_5_md.html"