Merge branch 'compression'

lib/httpx/options.rb

@@ -118,6 +118,12 @@ module HTTPX
 
     def dup
       dupped = super
+      dupped.headers             = headers.dup 
+      dupped.ssl                 = ssl.dup 
+      dupped.request_class       = request_class.dup 
+      dupped.response_class      = response_class.dup
+      dupped.headers_class       = headers_class.dup
+      dupped.response_body_class = response_body_class.dup
       yield(dupped) if block_given?
       dupped
     end

lib/httpx/plugins/compression.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins
+    module Compression
+      ACCEPT_ENCODING = %w[gzip deflate].freeze
+
+      def self.load_dependencies(*)
+        require "zlib"
+      end
+
+      module RequestMethods
+        def initialize(*)
+          super
+          ACCEPT_ENCODING.each do |enc|
+            @headers.add("accept-encoding", enc)
+          end
+        end
+      end
+
+      module ResponseBodyMethods
+        def initialize(*)
+          super
+          @_decoders = @headers.get("content-encoding").map do |encoding|
+            Transcoder.registry(encoding)
+          end
+        end
+
+        def write(*)
+          super
+          if @length == @headers["content-length"].to_i
+            @buffer.rewind
+            @buffer = decompress(@buffer)
+          end
+        end
+
+        private
+
+        def decompress(buffer)
+          @_decoders.reverse_each do |decoder|
+            buffer = decoder.decode(buffer)
+          end
+          buffer
+        end 
+      end
+
+      class CompressEncoder
+        attr_reader :content_type
+
+        def initialize(raw)
+          @content_type = raw.content_type
+          @raw = raw.respond_to?(:read) ? raw : StringIO.new(raw.to_s)
+          @buffer = StringIO.new("".b, File::RDWR)
+        end
+
+        def each(&blk)
+          return enum_for(__method__) unless block_given?
+          unless @buffer.size.zero?
+            @buffer.rewind
+            return @buffer.each(&blk)
+          end
+          compress(&blk)
+        end
+
+        def to_s
+          compress
+          @buffer.rewind
+          @buffer.read
+        end
+
+        def bytesize
+          compress
+          @buffer.size
+        end
+
+        def close
+          # @buffer.close
+        end
+      end
+
+      module GZIPTranscoder
+        class Encoder < CompressEncoder
+          def write(chunk)
+            @compressed_chunk = chunk
+          end
+
+          private
+
+          def compressed_chunk
+            compressed = @compressed_chunk
+            compressed
+          ensure
+            @compressed_chunk = nil
+          end
+
+          def compress
+            return unless @buffer.size.zero?
+            @raw.rewind
+            begin
+              gzip = Zlib::GzipWriter.new(self)
+
+              while chunk = @raw.read(16_384)
+                gzip.write(chunk)
+                gzip.flush
+                compressed = compressed_chunk
+                @buffer << compressed
+                yield compressed if block_given?
+              end
+            ensure
+              gzip.close
+            end
+          end
+        end
+       
+        module_function
+        
+        def encode(payload)
+          Encoder.new(payload)
+        end
+        
+        def decode(io)
+          Zlib::GzipReader.new(io, window_size: 32 + Zlib::MAX_WBITS)
+        end
+      end
+
+      module DeflateTranscoder
+        class Encoder < CompressEncoder
+          private
+
+          def compress
+            return unless @buffer.size.zero?
+            @raw.rewind
+            begin
+              deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,
+                                           Zlib::MAX_WBITS,
+                                           Zlib::MAX_MEM_LEVEL,
+                                           Zlib::HUFFMAN_ONLY)
+              while chunk = @raw.read(16_384)
+                compressed = deflater.deflate(chunk)
+                @buffer << compressed
+                yield compressed if block_given?
+              end
+              last = deflater.finish
+              @buffer << last
+              yield last if block_given?
+            ensure
+              deflater.close
+            end
+          end
+        end
+
+        module_function
+
+        class Decoder
+          def initialize(io)
+            @io = io
+            @inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
+            @buffer = StringIO.new
+          end
+         
+          def rewind
+            @buffer.rewind
+          end
+
+          def read(*args)
+            return @buffer.read(*args) if @io.eof?
+            chunk = @io.read(*args)
+            inflated_chunk = @inflater.inflate(chunk)
+            inflated_chunk << @inflater.finish if @io.eof?
+            @buffer << chunk
+            inflated_chunk
+          end
+
+          def close
+            @io.close
+            @io.unlink if @io.respond_to?(:unlink)
+            @inflater.close
+          end
+        end
+
+        def encode(payload)
+          Encoder.new(payload)
+        end
+
+        def decode(io)
+          Decoder.new(io)
+        end
+      end
+      Transcoder.register "gzip", GZIPTranscoder
+      Transcoder.register "deflate", DeflateTranscoder
+    end
+    register_plugin :compression, Compression
+  end
+
+
+end

lib/httpx/request.rb

@@ -109,6 +109,9 @@ module HTTPX
           Transcoder.registry("json").encode(options.json)
         end
         return if @body.nil?
+        @headers.get("content-encoding").each do |encoding|
+          @body = Transcoder.registry(encoding).encode(@body)
+        end
         @headers["content-type"] ||= @body.content_type
         @headers["content-length"] ||= @body.bytesize unless chunked?
       end

lib/httpx/response.rb

@@ -26,11 +26,12 @@ module HTTPX
       @request = request
       @status = Integer(status)
       @headers = @options.headers_class.new(headers)
-      @body = @options.response_body_class.new(self, threshold_size: @options.body_threshold_size)
+      @body = @options.response_body_class.new(self, threshold_size: @options.body_threshold_size,
+                                                     window_size: @options.window_size)
     end 
 
     def <<(data)
-      @body << data
+      @body.write(data)
     end
 
     def bodyless?
@@ -47,10 +48,11 @@ module HTTPX
     end
 
     class Body
-      def initialize(response, threshold_size: )
+      def initialize(response, threshold_size: , window_size: 1 << 14)
         @response = response
         @headers = response.headers
         @threshold_size = threshold_size
+        @window_size = window_size 
         @encoding = response.content_type.charset || Encoding::BINARY
         @length = 0
         @buffer = nil 
@@ -62,7 +64,6 @@ module HTTPX
         transition
         @buffer.write(chunk)
       end
-      alias :<< :write
 
       def read(*args)
         return unless @buffer
@@ -78,8 +79,8 @@ module HTTPX
         begin
           unless @state == :idle
             rewind
-            @buffer.each do |*args|
-              yield(*args)
+            while chunk = @buffer.read(@window_size)
+              yield(chunk)
             end
           end
         ensure

test/http1_test.rb

@@ -17,6 +17,7 @@ class HTTP1Test < HTTPTest
   include Plugins::Authentication
   include Plugins::FollowRedirects  
   include Plugins::Cookies
+  include Plugins::Compression
 
   private
 

test/http2_test.rb

@@ -15,6 +15,7 @@ class HTTP2Test < HTTPTest
   include Plugins::Authentication
   include Plugins::FollowRedirects
   include Plugins::Cookies
+  include Plugins::Compression
 
   private
 

test/response_test.rb

@@ -26,10 +26,10 @@ class ResponseTest < Minitest::Test
     opts = { threshold_size: 1024 }
     body1 = Response::Body.new(Response.new(request, 200, {}), opts)
     assert body1.empty?, "body must be empty after initialization"
-    body1 << "foo"
+    body1.write("foo")
     assert body1 == "foo", "body must be updated"
-    body1 << "foo"
-    body1 << "bar"
+    body1.write("foo")
+    body1.write("bar")
     assert body1 == "foobar", "body must buffer subsequent chunks"
 
     body3 = Response::Body.new(Response.new(request("head"), 200, {}), opts)
@@ -41,10 +41,10 @@ class ResponseTest < Minitest::Test
   def test_response_body_each
     opts = { threshold_size: 1024 }
     body1 = Response::Body.new(Response.new(request, 200, {}), opts)
-    body1 << "foo"
+    body1.write("foo")
     assert body1.each.to_a == %w(foo), "must yield buffer"
-    body1 << "foo"
-    body1 << "bar"
+    body1.write("foo")
+    body1.write("bar")
     assert body1.each.to_a == %w(foobar), "must yield buffers"
   end
 

test/support/requests/plugins/compression.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Requests
+  module Plugins
+    module Compression
+      def test_plugin_compression_accepts
+        url = "https://github.com"
+        response1 = HTTPX.get(url)
+        skip if response1.status == 429
+        verify_status(response1.status, 200)
+        assert !response1.headers.key?("content-encoding"), "response should come in plain text"
+
+        client = HTTPX.plugin(:compression)
+        response = client.get(url)
+        skip if response.status == 429
+        verify_status(response.status, 200)
+        verify_header(response.headers, "content-encoding", "gzip")
+      end
+
+      def test_plugin_compression_gzip
+        client = HTTPX.plugin(:compression)
+        uri = build_uri("/gzip")
+        response = client.get(uri)
+        verify_status(response.status, 200)
+        body = json_body(response)
+        assert body["gzipped"], "response should be gzipped"
+      end
+
+      def test_plugin_compression_gzip_post
+        client = HTTPX.plugin(:compression)
+        uri = build_uri("/post")
+        response = client.headers("content-encoding" => "gzip")
+                         .post(uri, body: "a" * 8012)
+        verify_status(response.status, 200)
+        body = json_body(response)
+        verify_header(body["headers"], "Content-Type", "application/octet-stream")
+        compressed_data = body["data"]
+        assert body["data"].bytesize < 8012, "body hasn't been compressed"
+      end
+
+      def test_plugin_compression_deflate
+        client = HTTPX.plugin(:compression)
+        uri = build_uri("/deflate")
+        response = client.get(uri)
+        verify_status(response.status, 200)
+        body = json_body(response)
+        assert body["deflated"], "response should be deflated"
+      end
+
+      def test_plugin_compression_deflate_post
+        client = HTTPX.plugin(:compression)
+        uri = build_uri("/post")
+        response = client.headers("content-encoding" => "deflate")
+                         .post(uri, body: "a" * 8012)
+        verify_status(response.status, 200)
+        body = json_body(response)
+        verify_header(body["headers"], "Content-Type", "application/octet-stream")
+        compressed_data = body["data"]
+        assert body["data"].bytesize < 8012, "body hasn't been compressed"
+      end
+
+    end
+  end
+end

test/support/requests/response_body.rb

@@ -41,7 +41,7 @@ module Requests
           @file = Tempfile.new
         end
 
-        def <<(data)
+        def write(data)
           @file << data
         end