mirror of
https://github.com/HoneyryderChuck/httpx.git
synced 2025-10-04 00:00:37 -04:00
Merge branch 'issue-341' into 'master'
response_cache plugin: return cached response from store unless stale Closes #341 See merge request os85/httpx!382
This commit is contained in:
HoneyryderChuck
commit
32c36bb4ee
@ -10,12 +10,14 @@ module HTTPX
|
||||
module ResponseCache
|
||||
CACHEABLE_VERBS = %w[GET HEAD].freeze
|
||||
CACHEABLE_STATUS_CODES = [200, 203, 206, 300, 301, 410].freeze
|
||||
SUPPORTED_VARY_HEADERS = %w[accept accept-encoding accept-language cookie origin].sort.freeze
|
||||
private_constant :CACHEABLE_VERBS
|
||||
private_constant :CACHEABLE_STATUS_CODES
|
||||
|
||||
class << self
|
||||
def load_dependencies(*)
|
||||
require_relative "response_cache/store"
|
||||
require_relative "response_cache/file_store"
|
||||
end
|
||||
|
||||
def cacheable_response?(response)
|
||||
@ -32,9 +34,7 @@ module HTTPX
|
||||
# directive prohibits caching. However, a cache that does not support
|
||||
# the Range and Content-Range headers MUST NOT cache 206 (Partial
|
||||
# Content) responses.
|
||||
response.status != 206 && (
|
||||
response.headers.key?("etag") || response.headers.key?("last-modified") || response.fresh?
|
||||
)
|
||||
response.status != 206
|
||||
end
|
||||
|
||||
def cached_response?(response)
|
||||
@ -42,15 +42,27 @@ module HTTPX
|
||||
end
|
||||
|
||||
def extra_options(options)
|
||||
options.merge(response_cache_store: Store.new)
|
||||
options.merge(
|
||||
supported_vary_headers: SUPPORTED_VARY_HEADERS,
|
||||
response_cache_store: :store,
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
module OptionsMethods
|
||||
def option_response_cache_store(value)
|
||||
raise TypeError, "must be an instance of #{Store}" unless value.is_a?(Store)
|
||||
case value
|
||||
when :store
|
||||
Store.new
|
||||
when :file_store
|
||||
FileStore.new
|
||||
else
|
||||
value
|
||||
end
|
||||
end
|
||||
|
||||
value
|
||||
def option_supported_vary_headers(value)
|
||||
Array(value).sort
|
||||
end
|
||||
end
|
||||
|
||||
@ -63,13 +75,19 @@ module HTTPX
|
||||
request = super
|
||||
return request unless cacheable_request?(request)
|
||||
|
||||
@options.response_cache_store.prepare(request)
|
||||
prepare_cache(request)
|
||||
|
||||
request
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def send_request(request, *)
|
||||
return request if request.response
|
||||
|
||||
super
|
||||
end
|
||||
|
||||
def fetch_response(request, *)
|
||||
response = super
|
||||
|
||||
@ -77,41 +95,123 @@ module HTTPX
|
||||
|
||||
if ResponseCache.cached_response?(response)
|
||||
log { "returning cached response for #{request.uri}" }
|
||||
cached_response = @options.response_cache_store.lookup(request)
|
||||
|
||||
response.copy_from_cached(cached_response)
|
||||
|
||||
else
|
||||
@options.response_cache_store.cache(request, response)
|
||||
response.copy_from_cached!
|
||||
elsif request.cacheable_verb? && ResponseCache.cacheable_response?(response)
|
||||
request.options.response_cache_store.set(request, response) unless response.cached?
|
||||
end
|
||||
|
||||
response
|
||||
end
|
||||
|
||||
def prepare_cache(request)
|
||||
cached_response = request.options.response_cache_store.get(request)
|
||||
|
||||
return unless cached_response && match_by_vary?(request, cached_response)
|
||||
|
||||
cached_response.body.rewind
|
||||
|
||||
if cached_response.fresh?
|
||||
cached_response = cached_response.dup
|
||||
cached_response.mark_as_cached!
|
||||
request.response = cached_response
|
||||
request.emit(:response, cached_response)
|
||||
return
|
||||
end
|
||||
|
||||
request.cached_response = cached_response
|
||||
|
||||
if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
|
||||
request.headers.add("if-modified-since", last_modified)
|
||||
end
|
||||
|
||||
if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
|
||||
request.headers.add("if-none-match", etag)
|
||||
end
|
||||
end
|
||||
|
||||
def cacheable_request?(request)
|
||||
request.cacheable_verb? &&
|
||||
(
|
||||
!request.headers.key?("cache-control") || !request.headers.get("cache-control").include?("no-store")
|
||||
)
|
||||
end
|
||||
|
||||
def match_by_vary?(request, response)
|
||||
vary = response.vary
|
||||
|
||||
return true unless vary
|
||||
|
||||
original_request = response.instance_variable_get(:@request)
|
||||
|
||||
if vary == %w[*]
|
||||
request.options.supported_vary_headers.each do |field|
|
||||
return false unless request.headers[field] == original_request.headers[field]
|
||||
end
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
vary.all? do |field|
|
||||
!original_request.headers.key?(field) || request.headers[field] == original_request.headers[field]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module RequestMethods
|
||||
attr_accessor :cached_response
|
||||
|
||||
def initialize(*)
|
||||
super
|
||||
@cached_response = nil
|
||||
end
|
||||
|
||||
def merge_headers(*)
|
||||
super
|
||||
@response_cache_key = nil
|
||||
end
|
||||
|
||||
def cacheable_verb?
|
||||
CACHEABLE_VERBS.include?(@verb)
|
||||
end
|
||||
|
||||
def response_cache_key
|
||||
@response_cache_key ||= Digest::SHA1.hexdigest("httpx-response-cache-#{@verb}-#{@uri}")
|
||||
@response_cache_key ||= begin
|
||||
keys = [@verb, @uri]
|
||||
|
||||
@options.supported_vary_headers.each do |field|
|
||||
value = @headers[field]
|
||||
|
||||
keys << value if value
|
||||
end
|
||||
Digest::SHA1.hexdigest("httpx-response-cache-#{keys.join("-")}")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
module ResponseMethods
|
||||
def copy_from_cached(other)
|
||||
# 304 responses do not have content-type, which are needed for decoding.
|
||||
@headers = @headers.class.new(other.headers.merge(@headers))
|
||||
def initialize(*)
|
||||
super
|
||||
@cached = false
|
||||
end
|
||||
|
||||
@body = other.body.dup
|
||||
def cached?
|
||||
@cached
|
||||
end
|
||||
|
||||
def mark_as_cached!
|
||||
@cached = true
|
||||
end
|
||||
|
||||
def copy_from_cached!
|
||||
cached_response = @request.cached_response
|
||||
|
||||
return unless cached_response
|
||||
|
||||
# 304 responses do not have content-type, which are needed for decoding.
|
||||
@headers = @headers.class.new(cached_response.headers.merge(@headers))
|
||||
|
||||
@body = cached_response.body.dup
|
||||
|
||||
@body.rewind
|
||||
end
|
||||
@ -121,6 +221,8 @@ module HTTPX
|
||||
if cache_control
|
||||
return false if cache_control.include?("no-cache")
|
||||
|
||||
return true if cache_control.include?("immutable")
|
||||
|
||||
# check age: max-age
|
||||
max_age = cache_control.find { |directive| directive.start_with?("s-maxage") }
|
||||
|
||||
@ -138,13 +240,13 @@ module HTTPX
|
||||
begin
|
||||
expires = Time.httpdate(@headers["expires"])
|
||||
rescue ArgumentError
|
||||
return true
|
||||
return false
|
||||
end
|
||||
|
||||
return (expires - Time.now).to_i.positive?
|
||||
end
|
||||
|
||||
true
|
||||
false
|
||||
end
|
||||
|
||||
def cache_control
|
||||
@ -163,7 +265,7 @@ module HTTPX
|
||||
@vary = begin
|
||||
return unless @headers.key?("vary")
|
||||
|
||||
@headers["vary"].split(/ *, */)
|
||||
@headers["vary"].split(/ *, */).map(&:downcase)
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
106
lib/httpx/plugins/response_cache/file_store.rb
Normal file
106
lib/httpx/plugins/response_cache/file_store.rb
Normal file
@ -0,0 +1,106 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require "pathname"
|
||||
|
||||
module HTTPX::Plugins
|
||||
module ResponseCache
|
||||
class FileStore
|
||||
CRLF = HTTPX::Connection::HTTP1::CRLF
|
||||
|
||||
attr_reader :dir
|
||||
|
||||
def initialize(dir = Dir.tmpdir)
|
||||
@dir = Pathname.new(dir).join("httpx-response-cache")
|
||||
|
||||
FileUtils.mkdir_p(@dir)
|
||||
end
|
||||
|
||||
def clear
|
||||
FileUtils.rm_rf(@dir)
|
||||
end
|
||||
|
||||
def get(request)
|
||||
path = file_path(request)
|
||||
|
||||
return unless File.exist?(path)
|
||||
|
||||
File.open(path, mode: File::RDONLY | File::BINARY) do |f|
|
||||
f.flock(File::Constants::LOCK_SH)
|
||||
|
||||
read_from_file(request, f)
|
||||
end
|
||||
end
|
||||
|
||||
def set(request, response)
|
||||
path = file_path(request)
|
||||
|
||||
file_exists = File.exist?(path)
|
||||
|
||||
mode = file_exists ? File::RDWR : File::CREAT | File::Constants::WRONLY
|
||||
|
||||
File.open(path, mode: mode | File::BINARY) do |f|
|
||||
f.flock(File::Constants::LOCK_EX)
|
||||
|
||||
if file_exists
|
||||
cached_response = read_from_file(request, f)
|
||||
|
||||
if cached_response
|
||||
next if cached_response == request.cached_response
|
||||
|
||||
cached_response.close
|
||||
|
||||
f.truncate(0)
|
||||
|
||||
f.rewind
|
||||
end
|
||||
end
|
||||
|
||||
# cache the response
|
||||
f << response.status << CRLF
|
||||
f << response.version << CRLF
|
||||
|
||||
response.headers.each do |field, value|
|
||||
f << field << ":" << value << CRLF
|
||||
end
|
||||
|
||||
f << CRLF
|
||||
|
||||
response.body.rewind
|
||||
|
||||
::IO.copy_stream(response.body, f)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def file_path(request)
|
||||
@dir.join(request.response_cache_key)
|
||||
end
|
||||
|
||||
def read_from_file(request, f)
|
||||
# if it's an empty file
|
||||
return if f.eof?
|
||||
|
||||
status = f.readline.delete_suffix!(CRLF)
|
||||
version = f.readline.delete_suffix!(CRLF)
|
||||
|
||||
headers = {}
|
||||
while (line = f.readline) != CRLF
|
||||
line.delete_suffix!(CRLF)
|
||||
sep_index = line.index(":")
|
||||
|
||||
field = line.byteslice(0..(sep_index - 1))
|
||||
value = line.byteslice((sep_index + 1)..-1)
|
||||
|
||||
headers[field] = value
|
||||
end
|
||||
|
||||
response = request.options.response_class.new(request, status, version, headers)
|
||||
|
||||
::IO.copy_stream(f, response.body)
|
||||
|
||||
response
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -12,80 +12,19 @@ module HTTPX::Plugins
|
||||
@store_mutex.synchronize { @store.clear }
|
||||
end
|
||||
|
||||
def lookup(request)
|
||||
responses = _get(request)
|
||||
|
||||
return unless responses
|
||||
|
||||
responses.find(&method(:match_by_vary?).curry(2)[request])
|
||||
end
|
||||
|
||||
def cached?(request)
|
||||
lookup(request)
|
||||
end
|
||||
|
||||
def cache(request, response)
|
||||
return unless request.cacheable_verb? && ResponseCache.cacheable_response?(response)
|
||||
|
||||
_set(request, response)
|
||||
end
|
||||
|
||||
def prepare(request)
|
||||
cached_response = lookup(request)
|
||||
|
||||
return unless cached_response
|
||||
|
||||
return unless match_by_vary?(request, cached_response)
|
||||
|
||||
if !request.headers.key?("if-modified-since") && (last_modified = cached_response.headers["last-modified"])
|
||||
request.headers.add("if-modified-since", last_modified)
|
||||
end
|
||||
|
||||
if !request.headers.key?("if-none-match") && (etag = cached_response.headers["etag"]) # rubocop:disable Style/GuardClause
|
||||
request.headers.add("if-none-match", etag)
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def match_by_vary?(request, response)
|
||||
vary = response.vary
|
||||
|
||||
return true unless vary
|
||||
|
||||
original_request = response.instance_variable_get(:@request)
|
||||
|
||||
return request.headers.same_headers?(original_request.headers) if vary == %w[*]
|
||||
|
||||
vary.all? do |cache_field|
|
||||
cache_field.downcase!
|
||||
!original_request.headers.key?(cache_field) || request.headers[cache_field] == original_request.headers[cache_field]
|
||||
end
|
||||
end
|
||||
|
||||
def _get(request)
|
||||
def get(request)
|
||||
@store_mutex.synchronize do
|
||||
responses = @store[request.response_cache_key]
|
||||
|
||||
return unless responses
|
||||
|
||||
responses.select! do |res|
|
||||
!res.body.closed? && res.fresh?
|
||||
end
|
||||
|
||||
responses
|
||||
@store[request.response_cache_key]
|
||||
end
|
||||
end
|
||||
|
||||
def _set(request, response)
|
||||
def set(request, response)
|
||||
@store_mutex.synchronize do
|
||||
responses = (@store[request.response_cache_key] ||= [])
|
||||
cached_response = @store[request.response_cache_key]
|
||||
|
||||
responses.reject! do |res|
|
||||
res.body.closed? || !res.fresh? || match_by_vary?(request, res)
|
||||
end
|
||||
cached_response.close if cached_response
|
||||
|
||||
responses << response
|
||||
@store[request.response_cache_key] = response
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -71,6 +71,14 @@ module HTTPX
|
||||
@content_type = nil
|
||||
end
|
||||
|
||||
# dupped initialization
|
||||
def initialize_dup(orig)
|
||||
super
|
||||
# if a response gets dupped, the body handle must also get dupped to prevent
|
||||
# two responses from using the same file handle to read.
|
||||
@body = orig.body.dup
|
||||
end
|
||||
|
||||
# closes the respective +@request+ and +@body+.
|
||||
def close
|
||||
@request.close
|
||||
|
||||
@ -11,6 +11,9 @@ module HTTPX
|
||||
# Array of encodings contained in the response "content-encoding" header.
|
||||
attr_reader :encodings
|
||||
|
||||
attr_reader :buffer
|
||||
protected :buffer
|
||||
|
||||
# initialized with the corresponding HTTPX::Response +response+ and HTTPX::Options +options+.
|
||||
def initialize(response, options)
|
||||
@response = response
|
||||
@ -148,13 +151,12 @@ module HTTPX
|
||||
end
|
||||
|
||||
def ==(other)
|
||||
object_id == other.object_id || begin
|
||||
if other.respond_to?(:read)
|
||||
_with_same_buffer_pos { FileUtils.compare_stream(@buffer, other) }
|
||||
else
|
||||
to_s == other.to_s
|
||||
end
|
||||
end
|
||||
super || case other
|
||||
when Response::Body
|
||||
@buffer == other.buffer
|
||||
else
|
||||
@buffer = other
|
||||
end
|
||||
end
|
||||
|
||||
# :nocov:
|
||||
@ -226,19 +228,6 @@ module HTTPX
|
||||
@state = nextstate
|
||||
end
|
||||
|
||||
def _with_same_buffer_pos # :nodoc:
|
||||
return yield unless @buffer && @buffer.respond_to?(:pos)
|
||||
|
||||
# @type ivar @buffer: StringIO | Tempfile
|
||||
current_pos = @buffer.pos
|
||||
@buffer.rewind
|
||||
begin
|
||||
yield
|
||||
ensure
|
||||
@buffer.pos = current_pos
|
||||
end
|
||||
end
|
||||
|
||||
class << self
|
||||
def initialize_inflater_by_encoding(encoding, response, **kwargs) # :nodoc:
|
||||
case encoding
|
||||
|
||||
@ -7,6 +7,9 @@ require "tempfile"
|
||||
module HTTPX
|
||||
# wraps and delegates to an internal buffer, which can be a StringIO or a Tempfile.
|
||||
class Response::Buffer < SimpleDelegator
|
||||
attr_reader :buffer
|
||||
protected :buffer
|
||||
|
||||
# initializes buffer with the +threshold_size+ over which the payload gets buffer to a tempfile,
|
||||
# the initial +bytesize+, and the +encoding+.
|
||||
def initialize(threshold_size:, bytesize: 0, encoding: Encoding::BINARY)
|
||||
@ -20,7 +23,14 @@ module HTTPX
|
||||
def initialize_dup(other)
|
||||
super
|
||||
|
||||
@buffer = other.instance_variable_get(:@buffer).dup
|
||||
# create new descriptor in READ-ONLY mode
|
||||
@buffer =
|
||||
case other.buffer
|
||||
when StringIO
|
||||
StringIO.new(other.buffer.string, mode: File::RDONLY)
|
||||
else
|
||||
other.buffer.class.new(other.buffer.path, encoding: Encoding::BINARY, mode: File::RDONLY)
|
||||
end
|
||||
end
|
||||
|
||||
# size in bytes of the buffered content.
|
||||
@ -46,7 +56,7 @@ module HTTPX
|
||||
end
|
||||
when Tempfile
|
||||
rewind
|
||||
content = _with_same_buffer_pos { @buffer.read }
|
||||
content = @buffer.read
|
||||
begin
|
||||
content.force_encoding(@encoding)
|
||||
rescue ArgumentError # ex: unknown encoding name - utf
|
||||
@ -61,6 +71,30 @@ module HTTPX
|
||||
@buffer.unlink if @buffer.respond_to?(:unlink)
|
||||
end
|
||||
|
||||
def ==(other)
|
||||
super || begin
|
||||
return false unless other.is_a?(Response::Buffer)
|
||||
|
||||
if @buffer.nil?
|
||||
other.buffer.nil?
|
||||
elsif @buffer.respond_to?(:read) &&
|
||||
other.respond_to?(:read)
|
||||
buffer_pos = @buffer.pos
|
||||
other_pos = other.buffer.pos
|
||||
@buffer.rewind
|
||||
other.buffer.rewind
|
||||
begin
|
||||
FileUtils.compare_stream(@buffer, other.buffer)
|
||||
ensure
|
||||
@buffer.pos = buffer_pos
|
||||
other.buffer.pos = other_pos
|
||||
end
|
||||
else
|
||||
to_s == other.to_s
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
# initializes the buffer into a StringIO, or turns it into a Tempfile when the threshold
|
||||
@ -82,15 +116,5 @@ module HTTPX
|
||||
|
||||
__setobj__(@buffer)
|
||||
end
|
||||
|
||||
def _with_same_buffer_pos # :nodoc:
|
||||
current_pos = @buffer.pos
|
||||
@buffer.rewind
|
||||
begin
|
||||
yield
|
||||
ensure
|
||||
@buffer.pos = current_pos
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
@ -3,30 +3,23 @@ module HTTPX
|
||||
module ResponseCache
|
||||
CACHEABLE_VERBS: Array[verb]
|
||||
CACHEABLE_STATUS_CODES: Array[Integer]
|
||||
SUPPORTED_VARY_HEADERS: Array[String]
|
||||
|
||||
def self?.cacheable_response?: (::HTTPX::ErrorResponse | (Response & ResponseMethods) response) -> bool
|
||||
def self?.cacheable_response?: (::HTTPX::ErrorResponse | cacheResponse response) -> bool
|
||||
def self?.cached_response?: (response response) -> bool
|
||||
|
||||
class Store
|
||||
@store: Hash[String, Array[Response]]
|
||||
interface _ResponseCacheOptions
|
||||
def response_cache_store: () -> Store
|
||||
|
||||
@store_mutex: Thread::Mutex
|
||||
def supported_vary_headers: () -> Array[String]
|
||||
end
|
||||
|
||||
def lookup: (Request request) -> Response?
|
||||
interface _ResponseCacheStore
|
||||
def get: (cacheRequest request) -> cacheResponse?
|
||||
|
||||
def cached?: (Request request) -> boolish
|
||||
def set: (cacheRequest request, cacheResponse response) -> void
|
||||
|
||||
def cache: (Request request, Response response) -> void
|
||||
|
||||
def prepare: (Request request) -> void
|
||||
|
||||
private
|
||||
|
||||
def match_by_vary?: (Request request, Response response) -> bool
|
||||
|
||||
def _get: (Request request) -> Array[Response]?
|
||||
|
||||
def _set: (Request request, Response response) -> void
|
||||
def clear: () -> void
|
||||
end
|
||||
|
||||
module InstanceMethods
|
||||
@ -34,10 +27,18 @@ module HTTPX
|
||||
|
||||
def clear_response_cache: () -> void
|
||||
|
||||
def cacheable_request?: (Request & RequestMethods request) -> bool
|
||||
private
|
||||
|
||||
def prepare_cache: (cacheRequest request) -> void
|
||||
|
||||
def cacheable_request?: (cacheRequest request) -> bool
|
||||
|
||||
def match_by_vary?: (cacheRequest request, cacheResponse response) -> bool
|
||||
end
|
||||
|
||||
module RequestMethods
|
||||
attr_accessor cached_response: cacheResponse?
|
||||
|
||||
@response_cache_key: String
|
||||
|
||||
def response_cache_key: () -> String
|
||||
@ -46,7 +47,13 @@ module HTTPX
|
||||
end
|
||||
|
||||
module ResponseMethods
|
||||
def copy_from_cached: (Response other) -> void
|
||||
@cache: bool
|
||||
|
||||
def cached?: () -> bool
|
||||
|
||||
def mark_as_cached!: () -> void
|
||||
|
||||
def copy_from_cached!: () -> void
|
||||
|
||||
def fresh?: () -> bool
|
||||
|
||||
@ -60,6 +67,13 @@ module HTTPX
|
||||
|
||||
def date: () -> Time
|
||||
end
|
||||
|
||||
|
||||
type cacheOptions = Options & _ResponseCacheOptions
|
||||
|
||||
type cacheRequest = Request & RequestMethods
|
||||
|
||||
type cacheResponse = Response & ResponseMethods
|
||||
end
|
||||
|
||||
type sessionResponseCache = Session & ResponseCache::InstanceMethods
|
||||
|
||||
@ -7,14 +7,15 @@ module HTTPX
|
||||
attr_reader encoding: Encoding
|
||||
attr_reader encodings: Array[String]
|
||||
|
||||
attr_reader buffer: Response::Buffer?
|
||||
|
||||
@response: Response
|
||||
@headers: Headers
|
||||
@options: Options
|
||||
@state: :idle | :memory | :buffer | :closed
|
||||
@window_size: Integer
|
||||
@length: Integer
|
||||
@buffer: StringIO | Tempfile | nil
|
||||
@reader: StringIO | Tempfile | nil
|
||||
@reader: Response::Buffer?
|
||||
@inflaters: Array[Transcoder::_Inflater]
|
||||
|
||||
def initialize: (Response, Options) -> void
|
||||
@ -47,7 +48,5 @@ module HTTPX
|
||||
def decode_chunk: (String chunk) -> String
|
||||
|
||||
def transition: (Symbol nextstate) -> void
|
||||
|
||||
def _with_same_buffer_pos: [A] () { () -> A } -> A
|
||||
end
|
||||
end
|
||||
@ -1,9 +1,10 @@
|
||||
module HTTPX
|
||||
class Response::Buffer
|
||||
attr_reader buffer: StringIO | Tempfile
|
||||
|
||||
@threshold_size: Integer
|
||||
@bytesize: Integer
|
||||
@encoding: Encoding
|
||||
@buffer: StringIO | Tempfile
|
||||
|
||||
def initialize: (threshold_size: Integer, ?bytesize: Integer, ?encoding: Encoding) -> void
|
||||
|
||||
@ -18,7 +19,5 @@ module HTTPX
|
||||
private
|
||||
|
||||
def try_upgrade_buffer: () -> void
|
||||
|
||||
def _with_same_buffer_pos: () { () -> void } -> void
|
||||
end
|
||||
end
|
||||
33
test/response_cache_file_store_test.rb
Normal file
33
test/response_cache_file_store_test.rb
Normal file
@ -0,0 +1,33 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "test_helper"
|
||||
require "httpx/plugins/response_cache/file_store"
|
||||
|
||||
class ResponseCacheFileStoreTest < Minitest::Test
|
||||
include ResponseCacheStoreTests
|
||||
|
||||
def test_internal_store_set
|
||||
request = make_request("GET", "http://store-set/")
|
||||
assert !File.exist?(store.dir.join(request.response_cache_key))
|
||||
cached_response(request)
|
||||
assert File.exist?(store.dir.join(request.response_cache_key))
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def store_class
|
||||
HTTPX::Plugins::ResponseCache::FileStore
|
||||
end
|
||||
|
||||
def store
|
||||
super.tap do |st|
|
||||
st.singleton_class.attr_writer :dir
|
||||
end
|
||||
end
|
||||
|
||||
def setup
|
||||
tmpdir = Pathname.new(Dir.tmpdir).join(SecureRandom.alphanumeric)
|
||||
FileUtils.mkdir_p(tmpdir)
|
||||
store.dir = tmpdir
|
||||
end
|
||||
end
|
||||
@ -4,137 +4,32 @@ require_relative "test_helper"
|
||||
require "httpx/plugins/response_cache/store"
|
||||
|
||||
class ResponseCacheStoreTest < Minitest::Test
|
||||
include HTTPX
|
||||
|
||||
def test_store_cache
|
||||
request = make_request("GET", "http://example.com/")
|
||||
response = cached_response(request)
|
||||
|
||||
assert store.lookup(request) == response
|
||||
assert store.cached?(request)
|
||||
|
||||
request2 = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
assert store.lookup(request2) == response
|
||||
|
||||
request3 = make_request("POST", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
assert store.lookup(request3).nil?
|
||||
end
|
||||
|
||||
def test_store_error_status
|
||||
request = make_request("GET", "http://example.com/")
|
||||
_response = cached_response(request, status: 404)
|
||||
assert !store.cached?(request)
|
||||
|
||||
_response = cached_response(request, status: 410)
|
||||
assert store.cached?(request)
|
||||
end
|
||||
|
||||
def test_store_no_store
|
||||
request = make_request("GET", "http://example.com/")
|
||||
_response = cached_response(request, extra_headers: { "cache-control" => "private, no-store" })
|
||||
assert !store.cached?(request)
|
||||
end
|
||||
|
||||
def test_store_maxage
|
||||
request = make_request("GET", "http://example.com/")
|
||||
response = cached_response(request, extra_headers: { "cache-control" => "max-age=2" })
|
||||
assert store.lookup(request) == response
|
||||
sleep(3)
|
||||
assert store.lookup(request).nil?
|
||||
|
||||
request2 = make_request("GET", "http://example2.com/")
|
||||
_response2 = cached_response(request2, extra_headers: { "cache-control" => "no-cache, max-age=2" })
|
||||
assert store.lookup(request2).nil?
|
||||
end
|
||||
|
||||
def test_store_expires
|
||||
request = make_request("GET", "http://example.com/")
|
||||
response = cached_response(request, extra_headers: { "expires" => (Time.now + 2).httpdate })
|
||||
assert store.lookup(request) == response
|
||||
sleep(3)
|
||||
assert store.lookup(request).nil?
|
||||
|
||||
request2 = make_request("GET", "http://example2.com/")
|
||||
cached_response(request2, extra_headers: { "cache-control" => "no-cache", "expires" => (Time.now + 2).httpdate })
|
||||
assert store.lookup(request2).nil?
|
||||
|
||||
request_invalid_expires = make_request("GET", "http://example3.com/")
|
||||
invalid_expires_response = cached_response(request_invalid_expires, extra_headers: { "expires" => "smthsmth" })
|
||||
assert store.lookup(request_invalid_expires) == invalid_expires_response
|
||||
end
|
||||
|
||||
def test_store_invalid_date
|
||||
request_invalid_age = make_request("GET", "http://example4.com/")
|
||||
response_invalid_age = cached_response(request_invalid_age, extra_headers: { "cache-control" => "max-age=2", "date" => "smthsmth" })
|
||||
assert store.lookup(request_invalid_age) == response_invalid_age
|
||||
end
|
||||
|
||||
def test_prepare_vary
|
||||
request = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
cached_response(request, extra_headers: { "vary" => "Accept" })
|
||||
|
||||
request2 = make_request("GET", "http://example.com/", headers: { "accept" => "text/html" })
|
||||
store.prepare(request2)
|
||||
assert !request2.headers.key?("if-none-match")
|
||||
request3 = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
store.prepare(request3)
|
||||
assert request3.headers.key?("if-none-match")
|
||||
request4 = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain", "user-agent" => "Linux Bowser" })
|
||||
store.prepare(request4)
|
||||
assert request4.headers.key?("if-none-match")
|
||||
end
|
||||
|
||||
def test_prepare_vary_asterisk
|
||||
request = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
cached_response(request, extra_headers: { "vary" => "*" })
|
||||
|
||||
request2 = make_request("GET", "http://example.com/", headers: { "accept" => "text/html" })
|
||||
store.prepare(request2)
|
||||
assert !request2.headers.key?("if-none-match")
|
||||
request3 = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain" })
|
||||
store.prepare(request3)
|
||||
assert request3.headers.key?("if-none-match")
|
||||
request4 = make_request("GET", "http://example.com/", headers: { "accept" => "text/plain", "user-agent" => "Linux Bowser" })
|
||||
store.prepare(request4)
|
||||
assert !request4.headers.key?("if-none-match")
|
||||
end
|
||||
include ResponseCacheStoreTests
|
||||
|
||||
def test_internal_store_set
|
||||
internal_store = store.instance_variable_get(:@store)
|
||||
|
||||
request = make_request("GET", "http://example.com/")
|
||||
|
||||
response = cached_response(request)
|
||||
assert internal_store[request.response_cache_key].size == 1
|
||||
assert internal_store[request.response_cache_key].include?(response)
|
||||
response1 = cached_response(request)
|
||||
assert internal_store[request.response_cache_key].size == 1
|
||||
assert internal_store[request.response_cache_key].include?(response1)
|
||||
response2 = cached_response(request, extra_headers: { "content-encoding" => "gzip" })
|
||||
assert internal_store[request.response_cache_key].size == 1
|
||||
assert internal_store[request.response_cache_key].include?(response2)
|
||||
assert internal_store.size == 1
|
||||
assert internal_store[request.response_cache_key] == response
|
||||
response1 = cached_response(request, extra_headers: { "content-language" => "en" })
|
||||
assert internal_store.size == 1
|
||||
assert internal_store[request.response_cache_key] == response1
|
||||
response2 = cached_response(request, extra_headers: { "content-language" => "en", "vary" => "accept-language" })
|
||||
assert internal_store.size == 1
|
||||
assert internal_store[request.response_cache_key] == response2
|
||||
|
||||
request.merge_headers("accept-language" => "pt")
|
||||
response3 = cached_response(request, extra_headers: { "content-language" => "pt", "vary" => "accept-language" }, body: "teste")
|
||||
assert internal_store.size == 2
|
||||
assert internal_store[request.response_cache_key] == response3
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def request_class
|
||||
@request_class ||= HTTPX.plugin(:response_cache).class.default_options.request_class
|
||||
end
|
||||
|
||||
def response_class
|
||||
@response_class ||= HTTPX.plugin(:response_cache).class.default_options.response_class
|
||||
end
|
||||
|
||||
def store
|
||||
@store ||= Plugins::ResponseCache::Store.new
|
||||
end
|
||||
|
||||
def make_request(meth, uri, *args)
|
||||
request_class.new(meth, uri, Options.new, *args)
|
||||
end
|
||||
|
||||
def cached_response(request, status: 200, extra_headers: {})
|
||||
response = response_class.new(request, status, "2.0", { "date" => Time.now.httpdate, "etag" => "ETAG" }.merge(extra_headers))
|
||||
store.cache(request, response)
|
||||
response
|
||||
def store_class
|
||||
Plugins::ResponseCache::Store
|
||||
end
|
||||
end
|
||||
|
||||
@ -25,22 +25,68 @@ module Requests
|
||||
|
||||
def test_plugin_response_cache_cache_control
|
||||
cache_client = HTTPX.plugin(:response_cache)
|
||||
# cache_control = 2
|
||||
|
||||
# cache_control_uri = build_uri("/cache/#{cache_control}")
|
||||
cache_control_uri = build_uri("/cache")
|
||||
|
||||
uncached = cache_client.get(cache_control_uri)
|
||||
verify_status(uncached, 200)
|
||||
cached = cache_client.get(cache_control_uri)
|
||||
verify_status(cached, 304)
|
||||
|
||||
assert uncached.body == cached.body
|
||||
# sleep(2)
|
||||
# expired = cache_client.get(cache_control_uri)
|
||||
# verify_status(expired, 200)
|
||||
end
|
||||
|
||||
# assert expired.body != uncached.body
|
||||
def test_plugin_response_cache_do_not_cache_on_error_status
|
||||
cache_client = HTTPX.plugin(SessionWithPool).plugin(:response_cache)
|
||||
|
||||
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
|
||||
|
||||
response_404 = cache_client.get(build_uri("/status/404"))
|
||||
verify_status(response_404, 404)
|
||||
assert !store.value?(response_404)
|
||||
|
||||
response_410 = cache_client.get(build_uri("/status/410"))
|
||||
verify_status(response_410, 410)
|
||||
assert store.value?(response_410)
|
||||
end
|
||||
|
||||
def test_plugin_response_cache_do_not_store_on_no_store_header
|
||||
return if origin.start_with?("https")
|
||||
|
||||
start_test_servlet(ResponseCacheServer) do |server|
|
||||
cache_client = HTTPX.plugin(:response_cache)
|
||||
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
|
||||
|
||||
response = cache_client.get("#{server.origin}/no-store")
|
||||
verify_status(response, 200)
|
||||
assert store.empty?, "request should not have been cached with no-store header"
|
||||
end
|
||||
end
|
||||
|
||||
def test_plugin_response_cache_return_cached_while_fresh
|
||||
cache_client = HTTPX.plugin(SessionWithPool).plugin(:response_cache)
|
||||
|
||||
cache_control_uri = build_uri("/cache/2")
|
||||
|
||||
store = cache_client.instance_variable_get(:@options).response_cache_store.instance_variable_get(:@store)
|
||||
|
||||
uncached = cache_client.get(cache_control_uri)
|
||||
verify_status(uncached, 200)
|
||||
assert cache_client.connection_count == 1, "a request should have been made"
|
||||
assert store.value?(uncached)
|
||||
|
||||
cached = cache_client.get(cache_control_uri)
|
||||
verify_status(cached, 200)
|
||||
assert cache_client.connection_count == 1, "no request should have been performed"
|
||||
assert uncached.body == cached.body, "bodies should have the same value"
|
||||
assert !uncached.body.eql?(cached.body), "bodies should have different references"
|
||||
assert store.value?(uncached)
|
||||
|
||||
sleep(2)
|
||||
after_expired = cache_client.get(cache_control_uri)
|
||||
verify_status(after_expired, 200)
|
||||
assert cache_client.connection_count == 2, "a conditional request should have been made"
|
||||
assert !store.value?(uncached)
|
||||
assert store.value?(after_expired)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
180
test/support/response_cache_store_tests.rb
Normal file
180
test/support/response_cache_store_tests.rb
Normal file
@ -0,0 +1,180 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
module ResponseCacheStoreTests
|
||||
include HTTPX
|
||||
def test_store_cache
|
||||
request = make_request("GET", "http://store-cache/")
|
||||
response = cached_response(request)
|
||||
|
||||
cached_response = store.get(request)
|
||||
assert cached_response.headers == response.headers
|
||||
assert cached_response.body == response.body
|
||||
assert store.get(request)
|
||||
|
||||
request2 = make_request("GET", "http://store-cache/")
|
||||
cached_response2 = store.get(request2)
|
||||
assert cached_response2
|
||||
assert cached_response2.headers == response.headers
|
||||
assert cached_response2.body == response.body
|
||||
|
||||
request3 = make_request("POST", "http://store-cache/")
|
||||
assert store.get(request3).nil?
|
||||
end
|
||||
|
||||
def test_store_prepare_maxage
|
||||
request = make_request("GET", "http://prepare-maxage/")
|
||||
response = cached_response(request, extra_headers: { "cache-control" => "max-age=2" })
|
||||
assert request.response.nil?
|
||||
|
||||
prepare(request)
|
||||
assert request.response
|
||||
assert request.response.headers == response.headers
|
||||
assert request.response.body == response.body
|
||||
assert request.cached_response.nil?
|
||||
|
||||
request.instance_variable_set(:@response, nil)
|
||||
sleep(3)
|
||||
|
||||
prepare(request)
|
||||
assert request.cached_response
|
||||
assert request.cached_response.headers == response.headers
|
||||
assert request.cached_response.body == response.body
|
||||
assert request.response.nil?
|
||||
|
||||
request2 = make_request("GET", "http://prepare-cache-2.com/")
|
||||
cached_response(request2, extra_headers: { "cache-control" => "no-cache, max-age=2" })
|
||||
prepare(request2)
|
||||
assert request2.response.nil?
|
||||
end
|
||||
|
||||
def test_store_prepare_expires
|
||||
request = make_request("GET", "http://prepare-expires/")
|
||||
response = cached_response(request, extra_headers: { "expires" => (Time.now + 2).httpdate })
|
||||
assert request.response.nil?
|
||||
|
||||
prepare(request)
|
||||
assert request.response
|
||||
assert request.response.headers == response.headers
|
||||
assert request.response.body == response.body
|
||||
assert request.cached_response.nil?
|
||||
|
||||
request.instance_variable_set(:@response, nil)
|
||||
sleep(3)
|
||||
|
||||
prepare(request)
|
||||
assert request.cached_response
|
||||
assert request.cached_response.headers == response.headers
|
||||
assert request.cached_response.body == response.body
|
||||
assert request.response.nil?
|
||||
|
||||
request2 = make_request("GET", "http://prepare-expires-2/")
|
||||
cached_response(request2, extra_headers: { "cache-control" => "no-cache", "expires" => (Time.now + 2).httpdate })
|
||||
prepare(request2)
|
||||
assert request2.response.nil?
|
||||
|
||||
request_invalid_expires = make_request("GET", "http://prepare-expires-3/")
|
||||
_invalid_expires_response = cached_response(request_invalid_expires, extra_headers: { "expires" => "smthsmth" })
|
||||
prepare(request_invalid_expires)
|
||||
assert request_invalid_expires.response.nil?
|
||||
end
|
||||
|
||||
def test_store_prepare_invalid_date
|
||||
request_invalid_age = make_request("GET", "http://prepare-expires-4/")
|
||||
response_invalid_age = cached_response(request_invalid_age, extra_headers: { "cache-control" => "max-age=2", "date" => "smthsmth" })
|
||||
prepare(request_invalid_age)
|
||||
assert request_invalid_age.response
|
||||
assert request_invalid_age.response.headers == response_invalid_age.headers
|
||||
assert request_invalid_age.response.body == response_invalid_age.body
|
||||
end
|
||||
|
||||
def test_prepare_vary
|
||||
request = make_request("GET", "http://prepare-vary/", headers: { "accept" => "text/plain" })
|
||||
response = cached_response(request, extra_headers: { "vary" => "Accept" })
|
||||
|
||||
request2 = make_request("GET", "http://prepare-vary/", headers: { "accept" => "text/html" })
|
||||
prepare(request2)
|
||||
assert !request2.headers.key?("if-none-match")
|
||||
assert request2.cached_response.nil?
|
||||
request3 = make_request("GET", "http://prepare-vary/", headers: { "accept" => "text/plain" })
|
||||
prepare(request3)
|
||||
assert request3.cached_response
|
||||
assert request3.cached_response.headers == response.headers
|
||||
assert request3.cached_response.body == response.body
|
||||
assert request3.headers.key?("if-none-match")
|
||||
request4 = make_request("GET", "http://prepare-vary/", headers: { "accept" => "text/plain", "user-agent" => "Linux Bowser" })
|
||||
prepare(request4)
|
||||
assert request4.cached_response
|
||||
assert request4.cached_response.headers == response.headers
|
||||
assert request4.cached_response.body == response.body
|
||||
assert request4.headers.key?("if-none-match")
|
||||
end
|
||||
|
||||
def test_prepare_vary_asterisk
|
||||
request = make_request("GET", "http://prepare-vary-asterisk/", headers: { "accept" => "text/plain" })
|
||||
response = cached_response(request, extra_headers: { "vary" => "*" })
|
||||
|
||||
request2 = make_request("GET", "http://prepare-vary-asterisk/", headers: { "accept" => "text/html" })
|
||||
prepare(request2)
|
||||
assert request.cached_response.nil?
|
||||
assert !request2.headers.key?("if-none-match")
|
||||
request3 = make_request("GET", "http://prepare-vary-asterisk/", headers: { "accept" => "text/plain" })
|
||||
prepare(request3)
|
||||
assert request3.cached_response
|
||||
assert request3.cached_response.headers == response.headers
|
||||
assert request3.cached_response.body == response.body
|
||||
assert request3.headers.key?("if-none-match")
|
||||
request4 = make_request("GET", "http://prepare-vary-asterisk/", headers: { "accept" => "text/plain", "accept-language" => "en" })
|
||||
prepare(request4)
|
||||
assert request4.cached_response.nil?
|
||||
assert !request4.headers.key?("if-none-match")
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def teardown
|
||||
response_cache_session.clear_response_cache
|
||||
end
|
||||
|
||||
def request_class
|
||||
@request_class ||= response_cache_session_options.request_class
|
||||
end
|
||||
|
||||
def response_class
|
||||
@response_class ||= response_cache_session_options.response_class
|
||||
end
|
||||
|
||||
def options_class
|
||||
@options_class ||= response_cache_session_optionsoptions_class
|
||||
end
|
||||
|
||||
def response_cache_session_options
|
||||
@response_cache_session_options ||= response_cache_session.class.default_options
|
||||
end
|
||||
|
||||
def response_cache_session
|
||||
@response_cache_session ||= HTTPX.plugin(:response_cache, response_cache_store: store_class.new)
|
||||
end
|
||||
|
||||
def make_request(meth, uri, *args)
|
||||
response_cache_session.build_request(meth, uri, *args)
|
||||
end
|
||||
|
||||
def store_class
|
||||
raise NotImplementedError, "must define a `store_class` method"
|
||||
end
|
||||
|
||||
def store
|
||||
response_cache_session.class.default_options.response_cache_store
|
||||
end
|
||||
|
||||
def cached_response(request, status: 200, extra_headers: {}, body: "test")
|
||||
response = response_class.new(request, status, "2.0", { "date" => Time.now.httpdate, "etag" => "ETAG" }.merge(extra_headers))
|
||||
response.body.write(body)
|
||||
store.set(request, response)
|
||||
response
|
||||
end
|
||||
|
||||
def prepare(request)
|
||||
response_cache_session.send(:prepare_cache, request)
|
||||
end
|
||||
end
|
||||
18
test/support/servlets/cache.rb
Normal file
18
test/support/servlets/cache.rb
Normal file
@ -0,0 +1,18 @@
|
||||
# frozen_string_literal: true
|
||||
|
||||
require_relative "test"
|
||||
|
||||
class ResponseCacheServer < TestServer
|
||||
class NoCacheApp < WEBrick::HTTPServlet::AbstractServlet
|
||||
def do_GET(_req, res) # rubocop:disable Naming/MethodName
|
||||
res.status = 200
|
||||
res.body = "no-cache"
|
||||
res["Cache-Control"] = "private, no-store"
|
||||
end
|
||||
end
|
||||
|
||||
def initialize(options = {})
|
||||
super
|
||||
mount("/no-store", NoCacheApp)
|
||||
end
|
||||
end
|
||||
Loading…
x
Reference in New Issue
Block a user