From 6336379837625db0a7ed223339cf37f41b2cfe7e Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Thu, 4 Aug 2022 23:17:24 +0100 Subject: [PATCH 1/3] added support for other json parsers --- lib/httpx/transcoder/json.rb | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/httpx/transcoder/json.rb b/lib/httpx/transcoder/json.rb index efdd105b..3affb004 100644 --- a/lib/httpx/transcoder/json.rb +++ b/lib/httpx/transcoder/json.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require "forwardable" -require "json" module HTTPX::Transcoder module JSON @@ -19,7 +18,7 @@ module HTTPX::Transcoder def_delegator :@raw, :bytesize def initialize(json) - @raw = ::JSON.dump(json) + @raw = json_dump(json) @charset = @raw.encoding.name.downcase end @@ -37,8 +36,25 @@ module HTTPX::Transcoder raise HTTPX::Error, "invalid json mime type (#{content_type})" unless JSON_REGEX.match?(content_type) - ::JSON.method(:parse) + method(:json_load) end + + # rubocop:disable Style/SingleLineMethods + if defined?(MultiJson) + def json_load(*args); MultiJson.load(*args); end + def json_dump(*args); MultiJson.dump(*args); end + elsif defined?(Oj) + def json_load(*args); Oj.load(*args); end + def json_dump(*args); Oj.dump(*args); end + elsif defined?(Yajl) + def json_load(*args); Yajl.load(*args); end + def json_dump(*args); Yajl.dump(*args); end + else + require "json" + def json_load(*args); ::JSON.parse(*args); end + def json_dump(*args); ::JSON.dump(*args); end + end + # rubocop:enable Style/SingleLineMethods end register "json", JSON end From 12573a16a55d014b6ed15ba1ac0b3f56f0ababad Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Aug 2022 09:20:11 +0100 Subject: [PATCH 2/3] removed support for application/dns-json mime type in the DoH resolver --- lib/httpx/resolver/https.rb | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/httpx/resolver/https.rb b/lib/httpx/resolver/https.rb index e7d3f27a..55d1f85e 100644 --- a/lib/httpx/resolver/https.rb +++ b/lib/httpx/resolver/https.rb @@ -102,7 +102,7 @@ module HTTPX @requests[request] = hostname resolver_connection.send(request) @connections << connection - rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e + rescue ResolveError, Resolv::DNS::EncodeError => e @queries.delete(hostname) emit_resolve_error(connection, connection.origin.host, e) end @@ -129,7 +129,7 @@ module HTTPX def parse(request, response) begin answers = decode_response_body(response) - rescue Resolv::DNS::DecodeError, JSON::JSONError => e + rescue Resolv::DNS::DecodeError => e host, connection = @queries.first @queries.delete(host) emit_resolve_error(connection, connection.origin.host, e) @@ -203,11 +203,6 @@ module HTTPX def decode_response_body(response) case response.headers["content-type"] - when "application/dns-json", - "application/json", - %r{^application/x-javascript} # because google... - payload = JSON.parse(response.to_s) - payload["Answer"] when "application/dns-udpwireformat", "application/dns-message" Resolver.decode_dns_answer(response.to_s) From 25b949cf66466c1b142e9cda58fa30aabdd7ec84 Mon Sep 17 00:00:00 2001 From: HoneyryderChuck Date: Fri, 5 Aug 2022 14:19:44 +0100 Subject: [PATCH 3/3] Added support for multiple JSON parsers When available, httpx will either use `multi_json`, `oj`, or `yajl`, before it falls back to default `json`. --- Gemfile | 24 ++++++++++------- lib/httpx/plugins/multipart/decoder.rb | 2 +- lib/httpx/response.rb | 8 +++--- lib/httpx/transcoder/form.rb | 2 +- lib/httpx/transcoder/json.rb | 8 +++--- sig/plugins/multipart.rbs | 2 +- sig/transcoder.rbs | 2 +- sig/transcoder/json.rbs | 3 +++ .../response_json_multi_json_test.rb | 27 +++++++++++++++++++ standalone_tests/response_json_oj_test.rb | 27 +++++++++++++++++++ standalone_tests/response_json_yajl_test.rb | 27 +++++++++++++++++++ 11 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 standalone_tests/response_json_multi_json_test.rb create mode 100644 standalone_tests/response_json_oj_test.rb create mode 100644 standalone_tests/response_json_yajl_test.rb diff --git a/Gemfile b/Gemfile index 5d30c31c..24287f0c 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,7 @@ group :test do gem "minitest" gem "minitest-proveit" gem "ruby-ntlm" - gem "sentry-ruby" if RUBY_VERSION >= "2.4" + gem "sentry-ruby" if RUBY_VERSION >= "2.4.0" gem "spy" if RUBY_VERSION < "2.3.0" gem "webmock", "< 3.15.0" @@ -25,16 +25,16 @@ group :test do end gem "websocket-driver" - gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2" + gem "net-ssh", "~> 4.2.0" if RUBY_VERSION < "2.2.0" - if RUBY_VERSION >= "2.3" + if RUBY_VERSION >= "2.3.0" gem "ddtrace" else - gem "ddtrace", "< 1.0" + gem "ddtrace", "< 1.0.0" end platform :mri do - if RUBY_VERSION >= "2.3" + if RUBY_VERSION >= "2.3.0" gem "google-protobuf", "< 3.19.2" if RUBY_VERSION < "2.5.0" gem "grpc" gem "logging" @@ -42,6 +42,12 @@ group :test do gem "mimemagic", require: false gem "ruby-filemagic", require: false end + + if RUBY_VERSION >= "3.0.0" + gem "multi_json", require: false + gem "oj", require: false + gem "yajl-ruby", require: false + end end platform :mri, :truffleruby do @@ -57,7 +63,7 @@ group :test do end platform :mri_23 do - if RUBY_VERSION >= "2.3" + if RUBY_VERSION >= "2.3.0" gem "openssl", "< 2.0.6" # force usage of openssl version we patch against end gem "msgpack", "<= 1.3.3" @@ -83,7 +89,7 @@ group :test do end group :coverage do - if RUBY_VERSION < "2.2" + if RUBY_VERSION < "2.2.0" gem "simplecov", "< 0.11.0" elsif RUBY_VERSION < "2.3" gem "simplecov", "< 0.11.0" @@ -109,14 +115,14 @@ group :website do end if RUBY_VERSION > "2.4" group :assorted do - if RUBY_VERSION < "2.2" + if RUBY_VERSION < "2.2.0" gem "pry", "~> 0.12.2" else gem "pry" end platform :mri do - if RUBY_VERSION < "2.2" + if RUBY_VERSION < "2.2.0" gem "pry-byebug", "~> 3.4.3" else gem "debug" if RUBY_VERSION >= "3.1.0" diff --git a/lib/httpx/plugins/multipart/decoder.rb b/lib/httpx/plugins/multipart/decoder.rb index 22e14373..c25ef0d5 100644 --- a/lib/httpx/plugins/multipart/decoder.rb +++ b/lib/httpx/plugins/multipart/decoder.rb @@ -61,7 +61,7 @@ module HTTPX::Plugins @state = :idle end - def call(response, _) + def call(response, *) response.body.each do |chunk| @buffer << chunk diff --git a/lib/httpx/response.rb b/lib/httpx/response.rb index 1245fa9e..658a5bd1 100644 --- a/lib/httpx/response.rb +++ b/lib/httpx/response.rb @@ -76,8 +76,8 @@ module HTTPX raise err end - def json(options = nil) - decode("json", options) + def json(*args) + decode("json", *args) end def form @@ -86,7 +86,7 @@ module HTTPX private - def decode(format, options = nil) + def decode(format, *args) # TODO: check if content-type is a valid format, i.e. "application/json" for json parsing transcoder = Transcoder.registry(format) @@ -96,7 +96,7 @@ module HTTPX raise Error, "no decoder available for \"#{format}\"" unless decoder - decoder.call(self, options) + decoder.call(self, *args) rescue Registry::Error raise Error, "no decoder available for \"#{format}\"" end diff --git a/lib/httpx/transcoder/form.rb b/lib/httpx/transcoder/form.rb index 8bca0e7d..adb01875 100644 --- a/lib/httpx/transcoder/form.rb +++ b/lib/httpx/transcoder/form.rb @@ -36,7 +36,7 @@ module HTTPX::Transcoder module Decoder module_function - def call(response, _) + def call(response, *) URI.decode_www_form(response.to_s).each_with_object({}) do |(field, value), params| HTTPX::Transcoder.normalize_query(params, field, value, PARAM_DEPTH_LIMIT) end diff --git a/lib/httpx/transcoder/json.rb b/lib/httpx/transcoder/json.rb index 3affb004..bf876fb9 100644 --- a/lib/httpx/transcoder/json.rb +++ b/lib/httpx/transcoder/json.rb @@ -18,7 +18,7 @@ module HTTPX::Transcoder def_delegator :@raw, :bytesize def initialize(json) - @raw = json_dump(json) + @raw = JSON.json_dump(json) @charset = @raw.encoding.name.downcase end @@ -44,11 +44,11 @@ module HTTPX::Transcoder def json_load(*args); MultiJson.load(*args); end def json_dump(*args); MultiJson.dump(*args); end elsif defined?(Oj) - def json_load(*args); Oj.load(*args); end + def json_load(response, *args); Oj.load(response.to_s, *args); end def json_dump(*args); Oj.dump(*args); end elsif defined?(Yajl) - def json_load(*args); Yajl.load(*args); end - def json_dump(*args); Yajl.dump(*args); end + def json_load(response, *args); Yajl::Parser.new(*args).parse(response.to_s); end + def json_dump(*args); Yajl::Encoder.encode(*args); end else require "json" def json_load(*args); ::JSON.parse(*args); end diff --git a/sig/plugins/multipart.rbs b/sig/plugins/multipart.rbs index cdeacbb9..be7d5867 100644 --- a/sig/plugins/multipart.rbs +++ b/sig/plugins/multipart.rbs @@ -61,7 +61,7 @@ module HTTPX @boundary: String @intermediate_boundary: String - def call: (Response response, untyped) -> Hash[String, untyped] + def call: (Response response, *untyped) -> Hash[String, untyped] private diff --git a/sig/transcoder.rbs b/sig/transcoder.rbs index 087de6d5..c3595b86 100644 --- a/sig/transcoder.rbs +++ b/sig/transcoder.rbs @@ -22,7 +22,7 @@ module HTTPX end interface _Decoder - def call: (Response response, untyped options) -> untyped + def call: (Response response, *untyped) -> untyped end end end diff --git a/sig/transcoder/json.rbs b/sig/transcoder/json.rbs index a53ccf7a..729a1049 100644 --- a/sig/transcoder/json.rbs +++ b/sig/transcoder/json.rbs @@ -5,6 +5,9 @@ module HTTPX::Transcoder def self?.encode: (_ToJson json) -> Encoder def self?.decode: (HTTPX::Response response) -> _Decoder + def self?.json_load: (string source, ?json_options) -> untyped + def self?.json_dump: (_ToJson obj, *untyped) -> String + class Encoder extend Forwardable include _Encoder diff --git a/standalone_tests/response_json_multi_json_test.rb b/standalone_tests/response_json_multi_json_test.rb new file mode 100644 index 00000000..c756f1a7 --- /dev/null +++ b/standalone_tests/response_json_multi_json_test.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "multi_json" +require "test_helper" + +class ResponseYajlTest < Minitest::Test + include HTTPX + + def test_response_decoders + json_response = Response.new(request, 200, "2.0", { "content-type" => "application/json" }) + json_response << %({"a": "b"}) + assert json_response.json == { "a" => "b" } + assert json_response.json(symbolize_keys: true) == { :a => "b" } + json_response << "bogus" + assert_raises(MultiJson::ParseError) { json_response.json } + end + + private + + def request(verb = :get, uri = "http://google.com") + Request.new(verb, uri) + end + + def response(*args) + Response.new(*args) + end +end diff --git a/standalone_tests/response_json_oj_test.rb b/standalone_tests/response_json_oj_test.rb new file mode 100644 index 00000000..726bd16d --- /dev/null +++ b/standalone_tests/response_json_oj_test.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "oj" +require "test_helper" + +class ResponseOjTest < Minitest::Test + include HTTPX + + def test_response_decoders + json_response = Response.new(request, 200, "2.0", { "content-type" => "application/json" }) + json_response << %({"a": "b"}) + assert json_response.json == { "a" => "b" } + assert json_response.json(symbol_keys: true) == { :a => "b" } + json_response << "bogus" + assert_raises(Oj::ParseError) { json_response.json } + end + + private + + def request(verb = :get, uri = "http://google.com") + Request.new(verb, uri) + end + + def response(*args) + Response.new(*args) + end +end diff --git a/standalone_tests/response_json_yajl_test.rb b/standalone_tests/response_json_yajl_test.rb new file mode 100644 index 00000000..ed8ff9d4 --- /dev/null +++ b/standalone_tests/response_json_yajl_test.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "yajl" +require "test_helper" + +class ResponseYajlTest < Minitest::Test + include HTTPX + + def test_response_decoders + json_response = Response.new(request, 200, "2.0", { "content-type" => "application/json" }) + json_response << %({"a": "b"}) + assert json_response.json == { "a" => "b" } + assert json_response.json(symbolize_keys: true) == { :a => "b" } + json_response << "bogus" + assert_raises(Yajl::ParseError) { json_response.json } + end + + private + + def request(verb = :get, uri = "http://google.com") + Request.new(verb, uri) + end + + def response(*args) + Response.new(*args) + end +end