sharpetronics/httpx
1
0
Fork 0
mirror of https://github.com/HoneyryderChuck/httpx.git synced 2025-08-10 00:01:27 -04:00
Code Issues Projects Releases Wiki Activity

simplifying apis, testing pathnames

Browse Source
This commit is contained in:
HoneyryderChuck 2021-01-11 01:18:48 +00:00
parent 478558d4bf
commit 42297cd38d
6 changed files with 217 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.simplecov
View File

@ -5,4 +5,5 @@ SimpleCov.start do
add_filter "/test/"
add_filter "/lib/httpx/extensions.rb"
add_filter "/lib/httpx/loggable.rb"
add_filter "/lib/httpx/plugins/multipart/mime_type_detector.rb"
end

197
lib/httpx/plugins/multipart.rb
View File

@ -17,192 +17,19 @@ module HTTPX
(value.key?(:filename) || value.key?(:content_type)))
end
def self.normalize_keys(key, value, &block)
Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
end
class Part
attr_reader :value
def initialize(key, value)
@key = key
@value = case value
when Hash
@content_type = value[:content_type]
@filename = value[:filename]
value[:body]
else
value
end
case @value
when Pathname
@value = @value.open(:binmode => true)
extract_from_file(@value)
when File
extract_from_file(@value)
when String
@value = StringIO.new(@value)
else
@filename ||= @value.filename if @value.respond_to?(:filename)
@content_type ||= @value.content_type if @value.respond_to?(:content_type)
raise Error, "#{@value} does not respond to #read#" unless @value.respond_to?(:read)
value
end
class << self
def normalize_keys(key, value, &block)
Transcoder.normalize_keys(key, value, MULTIPART_VALUE_COND, &block)
end
def header
header = "Content-Disposition: form-data; name=#{@key}".b
header << "; filename=#{@filename}" if @filename
header << "\r\n"
header << "Content-Type: #{@content_type}\r\n" if @content_type
header << "\r\n"
header
def load_dependencies(*)
require "httpx/plugins/multipart/encoder"
require "httpx/plugins/multipart/part"
require "httpx/plugins/multipart/mime_type_detector"
end
private
def extract_from_file(file)
@filename ||= File.basename(file.path)
@content_type ||= determine_mime_type(file) # rubocop:disable Naming/MemoizedInstanceVariableName
end
DEFAULT_MIMETYPE = "application/octet-stream"
# inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
if defined?(MIME::Types)
def determine_mime_type(_file)
mime = MIME::Types.of(@filename).first
return DEFAULT_MIMETYPE unless mime
mime.content_type
end
elsif defined?(MimeMagic)
def determine_mime_type(file)
mime = MimeMagic.by_magic(file)
return DEFAULT_MIMETYPE unless mime
return mime.type if mime
end
elsif system("which file", out: File::NULL)
require "open3"
def determine_mime_type(file)
return if file.eof? # file command returns "application/x-empty" for empty files
Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
begin
::IO.copy_stream(file, stdin.binmode)
rescue Errno::EPIPE
end
file.rewind
stdin.close
status = thread.value
# call to file command failed
if status.nil? || !status.success?
$stderr.print(stderr.read)
return DEFAULT_MIMETYPE
end
output = stdout.read.strip
if output.include?("cannot open")
$stderr.print(output)
return DEFAULT_MIMETYPE
end
output
end
end
else
def determine_mime_type(_file)
DEFAULT_MIMETYPE
end
end
end
class MultipartEncoder
def initialize(form)
@boundary = ("-" * 21) << SecureRandom.hex(21)
@part_index = 0
@buffer = "".b
@parts = to_parts(form)
end
def content_type
"multipart/form-data; boundary=#{@boundary}"
end
def bytesize
@parts.map(&:size).sum
end
def read(length = nil, outbuf = nil)
data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
data ||= "".b
read_chunks(data, length)
data unless length && data.empty?
end
private
def to_parts(form)
params = form.each_with_object([]) do |(key, val), aux|
Multipart.normalize_keys(key, val) do |k, v|
part = Part.new(k, v)
aux << StringIO.new("--#{@boundary}\r\n")
aux << StringIO.new(part.header)
aux << part.value
aux << StringIO.new("\r\n")
end
end
params << StringIO.new("--#{@boundary}--\r\n")
params
end
def read_chunks(buffer, length = nil)
while (chunk = read_from_part(length))
buffer << chunk.force_encoding(Encoding::BINARY)
next unless length
length -= chunk.bytesize
break if length.zero?
end
end
# if there's a current part to read from, tries to read a chunk.
def read_from_part(max_length = nil)
return unless @part_index < @parts.size
part = @parts[@part_index]
chunk = part.read(max_length, @buffer)
return chunk if chunk && !chunk.empty?
part.close
@part_index += 1
nil
def configure(*)
Transcoder.register("form", FormTranscoder)
end
end
@ -211,7 +38,7 @@ module HTTPX
def encode(form)
if multipart?(form)
MultipartEncoder.new(form)
Encoder.new(form)
else
Transcoder::Form::Encoder.new(form)
end
@ -225,10 +52,6 @@ module HTTPX
end
end
end
def self.configure(*)
Transcoder.register("form", FormTranscoder)
end
end
register_plugin :multipart, Multipart
end

84
lib/httpx/plugins/multipart/encoder.rb Normal file
View File

@ -0,0 +1,84 @@
# frozen_string_literal: true
module HTTPX::Plugins
module Multipart
class Encoder
def initialize(form)
@boundary = ("-" * 21) << SecureRandom.hex(21)
@part_index = 0
@buffer = "".b
@parts = to_parts(form)
end
def content_type
"multipart/form-data; boundary=#{@boundary}"
end
def bytesize
@parts.map(&:size).sum
end
def read(length = nil, outbuf = nil)
data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
data ||= "".b
read_chunks(data, length)
data unless length && data.empty?
end
private
def to_parts(form)
params = form.each_with_object([]) do |(key, val), aux|
Multipart.normalize_keys(key, val) do |k, v|
value, content_type, filename = Part.call(v)
aux << header_part(k, content_type, filename)
aux << value
aux << StringIO.new("\r\n")
end
end
params << StringIO.new("--#{@boundary}--\r\n")
params
end
def header_part(key, content_type, filename)
header = "--#{@boundary}\r\n".b
header << "Content-Disposition: form-data; name=#{key}".b
header << "; filename=#{filename}" if filename
header << "\r\nContent-Type: #{content_type}\r\n\r\n"
StringIO.new(header)
end
def read_chunks(buffer, length = nil)
while (chunk = read_from_part(length))
buffer << chunk.force_encoding(Encoding::BINARY)
next unless length
length -= chunk.bytesize
break if length.zero?
end
end
# if there's a current part to read from, tries to read a chunk.
def read_from_part(max_length = nil)
return unless @part_index < @parts.size
part = @parts[@part_index]
chunk = part.read(max_length, @buffer)
return chunk if chunk && !chunk.empty?
part.close if part.respond_to?(:close)
@part_index += 1
nil
end
end
end
end

64
lib/httpx/plugins/multipart/mime_type_detector.rb Normal file
View File

@ -0,0 +1,64 @@
# frozen_string_literal: true
module HTTPX
module Plugins::Multipart
module MimeTypeDetector
module_function
DEFAULT_MIMETYPE = "application/octet-stream"
# inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
if defined?(MIME::Types)
def call(_file, filename)
mime = MIME::Types.of(filename).first
mime.content_type if mime
end
elsif defined?(MimeMagic)
def call(file, *)
mime = MimeMagic.by_magic(file)
mime.type if mime
end
elsif system("which file", out: File::NULL)
require "open3"
def call(file, *)
return if file.eof? # file command returns "application/x-empty" for empty files
Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
begin
::IO.copy_stream(file, stdin.binmode)
rescue Errno::EPIPE
end
file.rewind
stdin.close
status = thread.value
# call to file command failed
if status.nil? || !status.success?
$stderr.print(stderr.read)
else
output = stdout.read.strip
if output.include?("cannot open")
$stderr.print(output)
else
output
end
end
end
end
else
def call(*); end
end
end
end
end

34
lib/httpx/plugins/multipart/part.rb Normal file
View File

@ -0,0 +1,34 @@
# frozen_string_literal: true
module HTTPX
module Plugins::Multipart
module Part
module_function
def call(value)
# take out specialized objects of the way
if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
return [value, value.content_type, value.filename]
end
content_type = filename = nil
if value.is_a?(Hash)
content_type = value[:content_type]
filename = value[:filename]
value = value[:body]
end
value = value.open(:binmode => true) if value.is_a?(Pathname)
if value.is_a?(File)
filename ||= File.basename(value.path)
content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
[value, content_type, filename]
else
[StringIO.new(value.to_s), "text/plain"]
end
end
end
end
end

26
test/support/requests/plugins/multipart.rb
View File

@ -118,6 +118,28 @@ module Requests
verify_uploaded_image(body, "q[image]", "image/jpeg")
end
define_method :"test_plugin_multipart_pathname_#{meth}" do
uri = build_uri("/#{meth}")
file = Pathname.new(fixture_file_path)
response = HTTPX.plugin(:multipart)
.send(meth, uri, form: { image: file })
verify_status(response, 200)
body = json_body(response)
verify_header(body["headers"], "Content-Type", "multipart/form-data")
verify_uploaded_image(body, "image", "image/jpeg")
end
define_method :"test_plugin_multipart_nested_pathname_#{meth}" do
uri = build_uri("/#{meth}")
file = Pathname.new(fixture_file_path)
response = HTTPX.plugin(:multipart)
.send(meth, uri, form: { q: { image: file } })
verify_status(response, 200)
body = json_body(response)
verify_header(body["headers"], "Content-Type", "multipart/form-data")
verify_uploaded_image(body, "q[image]", "image/jpeg")
end
define_method :"test_plugin_multipart_http_formdata_#{meth}" do
uri = build_uri("/#{meth}")
file = HTTP::FormData::File.new(fixture_file_path, content_type: "image/jpeg")
@ -126,7 +148,7 @@ module Requests
verify_status(response, 200)
body = json_body(response)
verify_header(body["headers"], "Content-Type", "multipart/form-data")
verify_uploaded_image(body, "image", file.content_type)
verify_uploaded_image(body, "image", "image/jpeg")
end
define_method :"test_plugin_multipart_nested_http_formdata_#{meth}" do
@ -137,7 +159,7 @@ module Requests
verify_status(response, 200)
body = json_body(response)
verify_header(body["headers"], "Content-Type", "multipart/form-data")
verify_uploaded_image(body, "q[image]", file.content_type)
verify_uploaded_image(body, "q[image]", "image/jpeg")
end
end