sharpetronics/httpx
1
0
Fork 0
mirror of https://github.com/HoneyryderChuck/httpx.git synced 2025-09-01 00:00:35 -04:00
Code Issues Projects Releases Wiki Activity

Merge branch 'multi_proxy' into 'master'

Browse Source 
Multi proxy

Closes #20

See merge request honeyryderchuck/httpx!22
This commit is contained in:
HoneyryderChuck 2018-06-11 18:25:31 +00:00
commit caadfd4024
4 changed files with 54 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
lib/httpx/plugins/proxy.rb
View File

@ -34,10 +34,15 @@ module HTTPX
private
def proxy_params(uri)
return @options.proxy if @options.proxy
uri = URI(uri).find_proxy
return unless uri
{ uri: uri }
@_proxy_uris ||= begin
uris = @options.proxy ? Array(@options.proxy[:uri]) : []
if uris.empty?
uri = URI(uri).find_proxy
uris << uri if uri
end
uris
end
@options.proxy.merge(uri: @_proxy_uris.shift) unless @_proxy_uris.empty?
end
def find_channel(request, **options)
@ -61,6 +66,21 @@ module HTTPX
@connection.__send__(:register_channel, channel)
channel
end
def fetch_response(request)
response = super
if response.is_a?(ErrorResponse) &&
# either it was a timeout error connecting, or it was a proxy error
((response.error.is_a?(TimeoutError) && request.state == :idle) ||
response.error.is_a?(Error)) &&
!@_proxy_uris.empty?
log { "failed connecting to proxy, trying next..." }
channel = find_channel(request)
channel.send(request)
return
end
response
end
end
module OptionsMethods
@ -91,6 +111,10 @@ module HTTPX
true
end
def send(request, **args)
@pending << [request, args]
end
def to_io
case @state
when :idle

2
lib/httpx/plugins/proxy/socks4.rb
View File

@ -53,7 +53,7 @@ module HTTPX
return unless @state == :connecting
@parser = nil
end
log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" }
log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
super
end
end

2
lib/httpx/plugins/proxy/socks5.rb
View File

@ -86,7 +86,7 @@ module HTTPX
return unless @state == :negotiating
@parser = nil
end
log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" }
log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
super
end

52
test/support/proxy_helper.rb
View File

@ -8,44 +8,40 @@ module ProxyHelper
private
def socks4_proxy
ENV["HTTPX_SOCKS4_PROXY"] || begin
ip, port, _, _ = socks_proxies_list.select do |_, _, version, https|
version == "Socks4" && https
end.sample
"socks4://#{ip}:#{port}"
end
Array(ENV["HTTPX_SOCKS4_PROXY"] || begin
socks_proxies_list.select { |_, _, version, https| version == "Socks4" && https }
.map { |ip, port, _, _| "socks4://#{ip}:#{port}" }
end)
end
def socks4a_proxy
ENV["HTTPX_SOCKS4A_PROXY"] || begin
ip, port, _, _ = socks_proxies_list.select do |_, _, version, https|
version == "Socks4" && https
end.sample
"socks4a://#{ip}:#{port}"
end
Array(ENV["HTTPX_SOCKS4A_PROXY"] || begin
socks_proxies_list.select { |_, _, version, https| version == "Socks4" && https }
.map { |ip, port, _, _| "socks4a://#{ip}:#{port}" }
end)
end
def socks5_proxy
ENV["HTTPX_SOCKS5_PROXY"] || begin
ip, port, _, _ = socks_proxies_list.select do |_, _, version, https|
version == "Socks5" && https
end.sample
"socks5://#{ip}:#{port}"
end
Array(ENV["HTTPX_SOCKS5_PROXY"] || begin
socks_proxies_list.select { |_, _, version, https| version == "Socks5" && https }
.map { |ip, port, _, _| "socks5://#{ip}:#{port}" }
end)
end
def http_proxy
ENV["HTTPX_HTTP_PROXY"] || begin
ip, port, _ = http_proxies_list.sample
"http://#{ip}:#{port}"
end
Array(ENV["HTTPX_HTTP_PROXY"] || begin
http_proxies_list.map do |ip, port, _|
"http://#{ip}:#{port}"
end
end)
end
def https_proxy
ENV["HTTPX_HTTPS_PROXY"] || begin
ip, port, _ = http_proxies_list.select { |_, _, https| https }.sample
"http://#{ip}:#{port}"
end
Array(ENV["HTTPX_HTTPS_PROXY"] || begin
http_proxies_list.select { |_, _, https| https }.map do |ip, port, _|
"http://#{ip}:#{port}"
end
end)
end
def http_proxies_list
@ -53,7 +49,7 @@ module ProxyHelper
.map do |line|
ip, port, _, _, _, _, https, _ = line.css("td").map(&:text)
[ip, port, https == "yes"]
end
end.select { |ip, port, _| ip && port } # rubocop:disable Style/MultilineBlockChain
end
def socks_proxies_list
@ -61,7 +57,7 @@ module ProxyHelper
.map do |line|
ip, port, _, _, version, _, https, _ = line.css("td").map(&:text)
[ip, port, version, https == "Yes"]
end
end.select { |ip, port, _, _| ip && port } # rubocop:disable Style/MultilineBlockChain
end
def proxies_list(document)