Merge branch 'content_digest' into 'master'

Add support for `content-digest` headers (RFC9530)

See merge request os85/httpx!354
This commit is contained in:
HoneyryderChuck 2024-12-02 12:37:40 +00:00
commit 85019e5493
10 changed files with 528 additions and 1 deletions

View File

@ -1,3 +1,3 @@
# Integration
This section is to test certain cases where we can't reliably reproduce in our test environments, but can be ran locally.
This section is to test certain cases where we can't reliably reproduce in our test environments, but can be ran locally.

View File

@ -0,0 +1,200 @@
# frozen_string_literal: true
module HTTPX
module Plugins
#
# This plugin adds `Content-Digest` headers to requests
# and can validate these headers on responses
#
# https://datatracker.ietf.org/doc/html/rfc9530
#
module ContentDigest
class Error < HTTPX::Error; end
# Error raised on response "content-digest" header validation.
class ValidationError < Error
attr_reader :response
def initialize(message, response)
super(message)
@response = response
end
end
class MissingContentDigestError < ValidationError; end
class InvalidContentDigestError < ValidationError; end
SUPPORTED_ALGORITHMS = {
"sha-256" => OpenSSL::Digest::SHA256,
"sha-512" => OpenSSL::Digest::SHA512,
}.freeze
class << self
def extra_options(options)
options.merge(encode_content_digest: true, validate_content_digest: true, content_digest_algorithm: "sha-256")
end
end
# add support for the following options:
#
# :content_digest_algorithm :: the digest algorithm to use. Currently supports `sha-256` and `sha-512`. (defaults to `sha-256`)
# :encode_content_digest :: whether a <tt>Content-Digest</tt> header should be computed for the request;
# can also be a callable object (i.e. <tt>->(req) { ... }</tt>, defaults to <tt>true</tt>)
# :validate_content_digest :: whether a <tt>Content-Digest</tt> header in the response should be validated;
# can also be a callable object (i.e. <tt>->(res) { ... }</tt>, defaults to <tt>true</tt>)
module OptionsMethods
def option_content_digest_algorithm(value)
raise TypeError, ":content_digest_algorithm must be one of 'sha-256', 'sha-512'" unless SUPPORTED_ALGORITHMS.key?(value)
value
end
def option_encode_content_digest(value)
value
end
def option_validate_content_digest(value)
value
end
end
module ResponseBodyMethods
attr_reader :content_digest_buffer
def initialize(response, options)
super
return unless response.headers.key?("content-digest")
should_validate = options.validate_content_digest
should_validate = should_validate.call(response) if should_validate.respond_to?(:call)
return unless should_validate
@content_digest_buffer = Response::Buffer.new(
threshold_size: @options.body_threshold_size,
bytesize: @length,
encoding: @encoding
)
end
def write(chunk)
@content_digest_buffer.write(chunk) if @content_digest_buffer
super
end
def close
if @content_digest_buffer
@content_digest_buffer.close
@content_digest_buffer = nil
end
super
end
end
module InstanceMethods
def build_request(*)
request = super
return request if request.headers.key?("content-digest")
perform_encoding = @options.encode_content_digest
perform_encoding = perform_encoding.call(request) if perform_encoding.respond_to?(:call)
return request unless perform_encoding
digest = base64digest(request.body)
request.headers.add("content-digest", "#{@options.content_digest_algorithm}=:#{digest}:")
request
end
private
def fetch_response(request, _, _)
response = super
return response unless response.is_a?(Response)
perform_validation = @options.validate_content_digest
perform_validation = perform_validation.call(response) if perform_validation.respond_to?(:call)
validate_content_digest(response) if perform_validation
response
rescue ValidationError => e
ErrorResponse.new(request, e)
end
def validate_content_digest(response)
content_digest_header = response.headers["content-digest"]
raise MissingContentDigestError.new("response is missing a `content-digest` header", response) unless content_digest_header
digests = extract_content_digests(content_digest_header)
included_algorithms = SUPPORTED_ALGORITHMS.keys & digests.keys
raise MissingContentDigestError.new("unsupported algorithms: #{digests.keys.join(", ")}", response) if included_algorithms.empty?
content_buffer = response.body.content_digest_buffer
included_algorithms.each do |algorithm|
digest = SUPPORTED_ALGORITHMS.fetch(algorithm).new
digest_received = digests[algorithm]
digest_computed =
if content_buffer.respond_to?(:to_path)
content_buffer.flush
digest.file(content_buffer.to_path).base64digest
else
digest.base64digest(content_buffer.to_s)
end
raise InvalidContentDigestError.new("#{algorithm} digest does not match content",
response) unless digest_received == digest_computed
end
end
def extract_content_digests(header)
header.split(",").to_h do |entry|
algorithm, digest = entry.split("=", 2)
raise Error, "#{entry} is an invalid digest format" unless algorithm && digest
[algorithm, digest.byteslice(1..-2)]
end
end
def base64digest(body)
digest = SUPPORTED_ALGORITHMS.fetch(@options.content_digest_algorithm).new
if body.respond_to?(:read)
if body.respond_to?(:to_path)
digest.file(body.to_path).base64digest
else
raise ContentDigestError, "request body must be rewindable" unless body.respond_to?(:rewind)
buffer = Tempfile.new("httpx", encoding: Encoding::BINARY, mode: File::RDWR)
begin
IO.copy_stream(body, buffer)
buffer.flush
digest.file(buffer.to_path).base64digest
ensure
body.rewind
buffer.close
buffer.unlink
end
end
else
raise ContentDigestError, "base64digest for endless enumerators is not supported" if body.unbounded_body?
buffer = "".b
body.each { |chunk| buffer << chunk }
digest.base64digest(buffer)
end
end
end
end
register_plugin :content_digest, ContentDigest
end
end

View File

@ -35,6 +35,7 @@ module HTTPX
| (:circuit_breaker, ?options) -> Plugins::sessionCircuitBreaker
| (:oauth, ?options) -> Plugins::sessionOAuth
| (:callbacks, ?options) -> Plugins::sessionCallbacks
| (:content_digest, ?options) -> Plugins::sessionContentDigest
| (Symbol | Module, ?options) { (Class) -> void } -> Session
| (Symbol | Module, ?options) -> Session

View File

@ -0,0 +1,51 @@
module HTTPX
module Plugins
module ContentDigest
class Error < HTTPX::Error
end
class ValidationError < Error
attr_reader response: Response
def initialize: (String message, Response response) -> void
end
class MissingContentDigestError < ValidationError
end
class InvalidContentDigestError < ValidationError
end
SUPPORTED_ALGORITHMS: Hash[String, singleton(OpenSSL::Digest)]
interface _ContentDigestOptions
def digest_algorithm: () -> String
def encode_content_digest: () -> (bool | ^(Request) -> boolish)
def validate_content_digest: () -> (bool | ^(contentDigestResponse) -> boolish)
end
# def self.extra_options: (Options) -> contentDigestOptions
module InstanceMethods
private
def validate_content_digest: (Response response) -> void
def extract_content_digests: (String) -> Hash[String, String]
def base64digest: (Request::Body | contentDigestResponseBody body) -> String
end
module ResponseMethods
attr_reader body: contentDigestResponseBody
end
module ResponseBodyMethods
attr_reader content_digest_buffer: Response::Buffer?
end
type contentDigestOptions = Options & _ContentDigestOptions
type contentDigestResponse = Response & ResponseMethods
type contentDigestResponseBody = Response::Body & ResponseBodyMethods
end
type sessionContentDigest = Session & ContentDigest::InstanceMethods
end
end

View File

@ -0,0 +1,91 @@
# frozen_string_literal: true
require_relative "support/http_helpers"
class HTTPXContentDigestTest < Minitest::Test
def test_plugin_content_digest_default_request
request = HTTPX.plugin(:content_digest)
.build_request(
"POST",
"http://domain.com",
body: StringIO.new("{\"hello\": \"world\"}\n")
)
expected_digest = "sha-256=:RK/0qy18MlBSVnWgjwz6lZEWjP/lF5HF9bvEF8FabDg=:"
assert request.headers["content-digest"] == expected_digest,
"expected: \"#{expected_digest}\", got \"#{request.headers["content-digest"]}\" inspected"
end
def test_plugin_content_digest_request_sha512
request = HTTPX.plugin(:content_digest, content_digest_algorithm: "sha-512")
.build_request(
"POST",
"http://domain.com",
body: StringIO.new("{\"hello\": \"world\"}\n")
)
expected_digest = "sha-512=:YMAam51Jz/jOATT6/zvHrLVgOYTGFy1d6GJiOHTohq4yP+pgk4vf2aCsyRZOtw8MjkM7iw7yZ/WkppmM44T3qg==:"
assert request.headers["content-digest"] == expected_digest,
"expected: \"#{expected_digest}\", got \"#{request.headers["content-digest"]}\" inspected"
end
def test_plugin_content_digest_from_json
request = HTTPX.plugin(:content_digest)
.build_request(
"POST",
"http://domain.com",
json: { hello: "world" }
)
# json is encoded without whitespace / newline, so digest differs from RFC
expected_digest = "sha-256=:k6I5cakU5erL8KjSUVTNownDwccvu5kU1Hxg88toFYg=:"
assert request.headers["content-digest"] == expected_digest,
"expected: \"#{expected_digest}\", got \"#{request.headers["content-digest"]}\" inspected"
end
def test_plugin_content_digest_from_file
json_file = File.open(File.expand_path("support/fixtures/hello_world.json", __dir__))
request = HTTPX.plugin(:content_digest)
.build_request(
"POST",
"http://domain.com",
body: json_file
)
json_file.close
expected_digest = "sha-256=:RK/0qy18MlBSVnWgjwz6lZEWjP/lF5HF9bvEF8FabDg=:"
assert request.headers["content-digest"] == expected_digest,
"expected: \"#{expected_digest}\", got \"#{request.headers["content-digest"]}\" inspected"
end
def test_plugin_content_digest_deflate
request = HTTPX.plugin(:content_digest)
.build_request(
"POST",
"http://domain.com",
headers: {
"content-encoding" => "deflate",
},
body: "{\"hello\": \"world\"}\n"
)
expected_digest = "sha-256=:2BPbFIfCAhjEJQF/2ifXfGqoq39DbqbVbk6H3Ann5sE=:"
assert request.headers["content-digest"] == expected_digest,
"expected: \"#{expected_digest}\", got \"#{request.headers["content-digest"]}\" inspected"
end
def test_plugin_content_digest_skip_digest
request = HTTPX.plugin(:content_digest, encode_content_digest: false)
.build_request(
"POST",
"http://domain.com",
body: "{\"hello\": \"world\"}\n"
)
assert request.headers["content-digest"].nil?
end
end

View File

@ -23,6 +23,7 @@ class HTTPTest < Minitest::Test
include Plugins::Authentication
include Plugins::OAuth
include Plugins::FollowRedirects
include Plugins::ContentDigest
include Plugins::Cookies
include Plugins::H2C
include Plugins::Retries

View File

@ -23,6 +23,7 @@ class HTTPSTest < Minitest::Test
include Plugins::Authentication
include Plugins::OAuth
include Plugins::FollowRedirects
include Plugins::ContentDigest
include Plugins::Cookies
include Plugins::PushPromise if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
include Plugins::Retries

View File

@ -0,0 +1 @@
{"hello": "world"}

View File

@ -0,0 +1,111 @@
# frozen_string_literal: true
module Requests
module Plugins
module ContentDigest
IGNORE_MISSING_HEADER = ->(res) { res.headers.key?("content-digest") }
def test_content_digest_missing_no_validation
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest, validate_content_digest: false)
%w[/no_content_digest /invalid_content_digest].each do |path|
response = http.get(server.origin + path)
verify_status(response, 200)
end
end
end
def test_content_digest_missing_validation_if_present
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
response = http.get("#{server.origin}/no_content_digest")
verify_status(response, 200)
end
end
def test_content_digest_missing_validation_always
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/no_content_digest")
verify_error_response(response, HTTPX::Plugins::ContentDigest::MissingContentDigestError)
end
end
def test_content_digest_present_validation_if_present
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
response = http.get("#{server.origin}/valid_content_digest")
verify_status(response, 200)
end
end
def test_content_digest_present_validation_always
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/valid_content_digest")
verify_status(response, 200)
end
end
def test_content_digest_invalid_validation_if_present
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest, validate_content_digest: IGNORE_MISSING_HEADER)
response = http.get("#{server.origin}/invalid_content_digest")
verify_error_response(response, HTTPX::Plugins::ContentDigest::InvalidContentDigestError)
end
end
def test_content_digest_invalid_validation_always
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/invalid_content_digest")
verify_error_response(response, HTTPX::Plugins::ContentDigest::InvalidContentDigestError)
end
end
def test_content_digest_multiple_validation_always
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/multiple_content_digests")
verify_status(response, 200)
end
end
def test_content_digest_gzip_encoding
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/gzip_content_digest")
verify_status(response, 200)
end
end
def test_content_digest_large_response_body
start_test_servlet(ContentDigestServer) do |server|
http = HTTPX.plugin(:content_digest)
response = http.get("#{server.origin}/large_body_content_digest")
verify_status(response, 200)
end
end
end
end
end

View File

@ -0,0 +1,70 @@
# frozen_string_literal: true
require_relative "test"
class ContentDigestServer < TestServer
class NoDigestApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "{\"hello\": \"world\"}"
res["Content-Type"] = "application/json"
end
end
class ValidDigestApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "{\"hello\": \"world\"}"
res["Content-Type"] = "application/json"
res["Content-Digest"] = "sha-256=:X48E9qOokqqrvdts8nOJRJN3OWDUoyWxBf7kbu9DBPE=:"
end
end
class InvalidDigestApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "{\"hello\": \"world\"}"
res["Content-Type"] = "application/json"
res["Content-Digest"] = "sha-256=:Y59F0rPplrrsweut9mPKSKM4PXEVpzXyCg8lcv0ECQF=:"
end
end
class MultipleDigestsApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "{\"hello\": \"world\"}"
res["Content-Type"] = "application/json"
digest256 = "sha-256=:X48E9qOokqqrvdts8nOJRJN3OWDUoyWxBf7kbu9DBPE=:"
digest512 = "sha-512:WZDPaVn/7XgHaAy8pmojAkGWoRx2UFChF41A2svX+TaPm+AbwAgBWnrIiYllu7BNNyealdVLvRwEmTHWXvJwew==:"
res["Content-Digest"] = [digest256, digest512].join(",")
end
end
class GzipDigestApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "\x1F\x8B\b\x00\xBB\xD6Eg\x00\x03\xABV\xCAH\xCD\xC9\xC9W\xB2RP*\xCF/\xCAIQ\xAA\x05\x00\"\xAE\xA3\x86\x12\x00\x00\x00"
res["Content-Type"] = "application/json"
res["Content-Encoding"] = "gzip"
res["Content-Digest"] = "sha-256=:oswx8nqtHLL4Gky0pTtr8lKNF/IYNtAA4OUjONh+0Ns=:"
end
end
class LargeBodyApp < WEBrick::HTTPServlet::AbstractServlet
def do_GET(_req, res) # rubocop:disable Naming/MethodName
res.status = 200
res.body = "a#{"a" * HTTPX::Options::MAX_BODY_THRESHOLD_SIZE}"
res["Content-Digest"] = "sha-256=:#{OpenSSL::Digest.base64digest("sha256", res.body)}:"
end
end
def initialize(options = {})
super
mount("/no_content_digest", NoDigestApp)
mount("/valid_content_digest", ValidDigestApp)
mount("/invalid_content_digest", InvalidDigestApp)
mount("/multiple_content_digests", MultipleDigestsApp)
mount("/gzip_content_digest", GzipDigestApp)
mount("/large_body_content_digest", LargeBodyApp)
end
end