ProjectSegfault/invidious-experimenting
4
0
Fork 0
forked from midou/invidious
Code Pull Requests Activity

Add support for CONNECT proxy

Browse Source
This commit is contained in:
Omar Roth 2019-11-09 14:21:31 -05:00
parent ceb252986e
commit 50bab26a3a
No known key found for this signature in database
GPG Key ID: B8254FB7EC3D37F2
8 changed files with 170 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
src/invidious.cr
View File

@ -5568,7 +5568,7 @@ get "/videoplayback" do |env|
next env.redirect location
end
IO.copy(response.body_io, env.response)
IO.copy response.body_io, env.response
end
rescue ex
end
@ -5865,6 +5865,69 @@ get "/Captcha" do |env|
response.body
end
connect "*" do |env|
if CONFIG.proxy_address.empty?
env.response.status_code = 400
next
end
url = env.request.headers["Host"]?.try { |u| u.split(":") }
host = url.try &.[0]?
port = url.try &.[1]?
host = "www.google.com" if !host || host.empty?
port = "443" if !port || port.empty?
# if env.request.internal_uri
# env.request.internal_uri.not_nil!.path = "#{host}:#{port}"
# end
user, pass = env.request.headers["Proxy-Authorization"]?
.try { |i| i.lchop("Basic ") }
.try { |i| Base64.decode_string(i) }
.try &.split(":", 2) || {nil, nil}
if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass
env.response.status_code = 403
next
end
begin
upstream = TCPSocket.new(host, port)
rescue ex
logger.puts("Exception: #{ex.message}")
env.response.status_code = 400
next
end
env.response.reset
env.response.upgrade do |downstream|
downstream = downstream.as(TCPSocket)
downstream.sync = true
spawn do
begin
bytes = 1
while bytes != 0
bytes = IO.copy upstream, downstream
end
rescue ex
end
end
begin
bytes = 1
while bytes != 0
bytes = IO.copy downstream, upstream
end
rescue ex
ensure
upstream.close
downstream.close
end
end
end
# Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
get "/watch_videos" do |env|
response = YT_POOL.client &.get(env.request.resource)
@ -5939,6 +6002,7 @@ end
public_folder "assets"
Kemal.config.powered_by_header = false
add_handler ProxyHandler.new
add_handler FilteredCompressHandler.new
add_handler APIHandler.new
add_handler AuthHandler.new

29
src/invidious/helpers/handlers.cr
View File

@ -212,3 +212,32 @@ class DenyFrame < Kemal::Handler
call_next env
end
end
class ProxyHandler < Kemal::Handler
def call(env)
if env.request.headers["Proxy-Authorization"]? && env.request.method != "CONNECT"
user, pass = env.request.headers["Proxy-Authorization"]?
.try { |i| i.lchop("Basic ") }
.try { |i| Base64.decode_string(i) }
.try &.split(":", 2) || {nil, nil}
if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass
env.response.status_code = 403
return
end
HTTP::Client.exec(env.request.method, "#{env.request.headers["Host"]?}#{env.request.resource}", env.request.headers, env.request.body) do |response|
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "transfer-encoding"
env.response.headers[key] = value
end
end
IO.copy response.body_io, env.response
end
env.response.close
return
else
call_next env
end
end
end

4
src/invidious/helpers/helpers.cr
View File

@ -263,6 +263,10 @@ struct Config
admin_email: {type: String, default: "omarroth@protonmail.com"}, # Email for bug reports
cookies: {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies}, # Saved cookies in "name1=value1; name2=value2..." format
captcha_key: {type: String?, default: nil}, # Key for Anti-Captcha
proxy_address: {type: String, default: ""},
proxy_port: {type: Int32, default: 8080},
proxy_user: {type: String, default: ""},
proxy_pass: {type: String, default: ""},
})
end

82
src/invidious/helpers/jobs.cr
View File

@ -249,15 +249,34 @@ def bypass_captcha(captcha_key, logger)
end
headers = response.cookies.add_request_headers(HTTP::Headers.new)
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
"websiteKey" => site_key,
},
}.to_json).body)
captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
if !CONFIG.proxy_address.empty?
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTask",
"websiteURL" => "https://www.youtube.com#{path}",
"websiteKey" => site_key,
"proxyType" => "http",
"proxyAddress" => CONFIG.proxy_address,
"proxyPort" => CONFIG.proxy_port,
"proxyLogin" => CONFIG.proxy_user,
"proxyPassword" => CONFIG.proxy_pass,
"userAgent" => headers["user-agent"],
},
}.to_json).body)
else
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => "https://www.youtube.com#{path}",
"websiteKey" => site_key,
"userAgent" => headers["user-agent"],
},
}.to_json).body)
end
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
@ -265,7 +284,7 @@ def bypass_captcha(captcha_key, logger)
loop do
sleep 10.seconds
response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
response = JSON.parse(captcha_client.post("/getTaskResult", body: {
"clientKey" => CONFIG.captcha_key,
"taskId" => task_id,
}.to_json).body)
@ -283,7 +302,11 @@ def bypass_captcha(captcha_key, logger)
yield response.cookies.select { |cookie| cookie.name != "PREF" }
elsif response.headers["Location"]?.try &.includes?("/sorry/index")
location = response.headers["Location"].try { |u| URI.parse(u) }
headers = HTTP::Headers{":authority" => location.host.not_nil!}
headers = HTTP::Headers{
":authority" => location.host.not_nil!,
"origin" => "https://www.google.com",
"user-agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36",
}
response = YT_POOL.client &.get(location.full_path, headers)
html = XML.parse_html(response.body)
@ -297,14 +320,32 @@ def bypass_captcha(captcha_key, logger)
captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
},
}.to_json).body)
if !CONFIG.proxy_address.empty?
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTask",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
"proxyType" => "http",
"proxyAddress" => CONFIG.proxy_address,
"proxyPort" => CONFIG.proxy_port,
"proxyLogin" => CONFIG.proxy_user,
"proxyPassword" => CONFIG.proxy_pass,
"userAgent" => headers["user-agent"],
},
}.to_json).body)
else
response = JSON.parse(captcha_client.post("/createTask", body: {
"clientKey" => CONFIG.captcha_key,
"task" => {
"type" => "NoCaptchaTaskProxyless",
"websiteURL" => location.to_s,
"websiteKey" => site_key,
"userAgent" => headers["user-agent"],
},
}.to_json).body)
end
raise response["error"].as_s if response["error"]?
task_id = response["taskId"].as_i
@ -326,8 +367,7 @@ def bypass_captcha(captcha_key, logger)
inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
headers["content-type"] = "application/x-www-form-urlencoded"
headers["origin"] = "https://www.google.com"
headers["user-agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
headers["referer"] = location.to_s
response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
headers = HTTP::Headers{

8
src/invidious/helpers/proxy.cr
View File

@ -1,3 +1,7 @@
def connect(path : String, &block : HTTP::Server::Context -> _)
Kemal::RouteHandler::INSTANCE.add_route("CONNECT", path, &block)
end
# See https://github.com/crystal-lang/crystal/issues/2963
class HTTPProxy
getter proxy_host : String
@ -124,7 +128,7 @@ def get_nova_proxies(country_code = "US")
client.connect_timeout = 10.seconds
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
headers["Host"] = "www.proxynova.com"
@ -161,7 +165,7 @@ def get_spys_proxies(country_code = "US")
client.connect_timeout = 10.seconds
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
headers["Host"] = "spys.one"

5
src/invidious/helpers/utils.cr
View File

@ -2,11 +2,12 @@ require "lsquic"
require "pool/connection"
def add_yt_headers(request)
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
return if request.resource.starts_with? "/sorry/index"
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
return if request.resource.starts_with? "/sorry/index"
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "1.20180719"
if !CONFIG.cookies.empty?

2
src/invidious/mixes.cr
View File

@ -20,7 +20,7 @@ end
def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
if cookies
headers = cookies.add_request_headers(headers)

2
src/invidious/trending.cr
View File

@ -1,6 +1,6 @@
def fetch_trending(trending_type, region, locale)
headers = HTTP::Headers.new
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
region ||= "US"
region = region.upcase