Add support for CONNECT proxy

src/invidious.cr

@@ -5568,7 +5568,7 @@ get "/videoplayback" do |env|
           next env.redirect location
         end
 
-        IO.copy(response.body_io, env.response)
+        IO.copy response.body_io, env.response
       end
     rescue ex
     end
@@ -5865,6 +5865,69 @@ get "/Captcha" do |env|
   response.body
 end
 
+connect "*" do |env|
+  if CONFIG.proxy_address.empty?
+    env.response.status_code = 400
+    next
+  end
+
+  url = env.request.headers["Host"]?.try { |u| u.split(":") }
+  host = url.try &.[0]?
+  port = url.try &.[1]?
+
+  host = "www.google.com" if !host || host.empty?
+  port = "443" if !port || port.empty?
+
+  # if env.request.internal_uri
+  #   env.request.internal_uri.not_nil!.path = "#{host}:#{port}"
+  # end
+
+  user, pass = env.request.headers["Proxy-Authorization"]?
+    .try { |i| i.lchop("Basic ") }
+    .try { |i| Base64.decode_string(i) }
+    .try &.split(":", 2) || {nil, nil}
+
+  if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass
+    env.response.status_code = 403
+    next
+  end
+
+  begin
+    upstream = TCPSocket.new(host, port)
+  rescue ex
+    logger.puts("Exception: #{ex.message}")
+    env.response.status_code = 400
+    next
+  end
+
+  env.response.reset
+  env.response.upgrade do |downstream|
+    downstream = downstream.as(TCPSocket)
+    downstream.sync = true
+
+    spawn do
+      begin
+        bytes = 1
+        while bytes != 0
+          bytes = IO.copy upstream, downstream
+        end
+      rescue ex
+      end
+    end
+
+    begin
+      bytes = 1
+      while bytes != 0
+        bytes = IO.copy downstream, upstream
+      end
+    rescue ex
+    ensure
+      upstream.close
+      downstream.close
+    end
+  end
+end
+
 # Undocumented, creates anonymous playlist with specified 'video_ids', max 50 videos
 get "/watch_videos" do |env|
   response = YT_POOL.client &.get(env.request.resource)
@@ -5939,6 +6002,7 @@ end
 public_folder "assets"
 
 Kemal.config.powered_by_header = false
+add_handler ProxyHandler.new
 add_handler FilteredCompressHandler.new
 add_handler APIHandler.new
 add_handler AuthHandler.new

src/invidious/helpers/handlers.cr

@@ -212,3 +212,32 @@ class DenyFrame < Kemal::Handler
     call_next env
   end
 end
+
+class ProxyHandler < Kemal::Handler
+  def call(env)
+    if env.request.headers["Proxy-Authorization"]? && env.request.method != "CONNECT"
+      user, pass = env.request.headers["Proxy-Authorization"]?
+        .try { |i| i.lchop("Basic ") }
+        .try { |i| Base64.decode_string(i) }
+        .try &.split(":", 2) || {nil, nil}
+
+      if CONFIG.proxy_user != user || CONFIG.proxy_pass != pass
+        env.response.status_code = 403
+        return
+      end
+
+      HTTP::Client.exec(env.request.method, "#{env.request.headers["Host"]?}#{env.request.resource}", env.request.headers, env.request.body) do |response|
+        response.headers.each do |key, value|
+          if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase) && key.downcase != "transfer-encoding"
+            env.response.headers[key] = value
+          end
+        end
+        IO.copy response.body_io, env.response
+      end
+      env.response.close
+      return
+    else
+      call_next env
+    end
+  end
+end

src/invidious/helpers/helpers.cr

@@ -263,6 +263,10 @@ struct Config
     admin_email:       {type: String, default: "omarroth@protonmail.com"},                                  # Email for bug reports
     cookies:           {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies},       # Saved cookies in "name1=value1; name2=value2..." format
     captcha_key:       {type: String?, default: nil},                                                       # Key for Anti-Captcha
+    proxy_address:     {type: String, default: ""},
+    proxy_port:        {type: Int32, default: 8080},
+    proxy_user:        {type: String, default: ""},
+    proxy_pass:        {type: String, default: ""},
   })
 end
 

src/invidious/helpers/jobs.cr

@@ -249,15 +249,34 @@ def bypass_captcha(captcha_key, logger)
           end
 
           headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
+          captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
-          response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
+          captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
-            "clientKey" => CONFIG.captcha_key,
+          if !CONFIG.proxy_address.empty?
-            "task"      => {
+            response = JSON.parse(captcha_client.post("/createTask", body: {
-              "type"       => "NoCaptchaTaskProxyless",
+              "clientKey" => CONFIG.captcha_key,
-              "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
+              "task"      => {
-              "websiteKey" => site_key,
+                "type"          => "NoCaptchaTask",
-            },
+                "websiteURL"    => "https://www.youtube.com#{path}",
-          }.to_json).body)
+                "websiteKey"    => site_key,
+                "proxyType"     => "http",
+                "proxyAddress"  => CONFIG.proxy_address,
+                "proxyPort"     => CONFIG.proxy_port,
+                "proxyLogin"    => CONFIG.proxy_user,
+                "proxyPassword" => CONFIG.proxy_pass,
+                "userAgent"     => headers["user-agent"],
+              },
+            }.to_json).body)
+          else
+            response = JSON.parse(captcha_client.post("/createTask", body: {
+              "clientKey" => CONFIG.captcha_key,
+              "task"      => {
+                "type"       => "NoCaptchaTaskProxyless",
+                "websiteURL" => "https://www.youtube.com#{path}",
+                "websiteKey" => site_key,
+                "userAgent"  => headers["user-agent"],
+              },
+            }.to_json).body)
+          end
 
           raise response["error"].as_s if response["error"]?
           task_id = response["taskId"].as_i
@@ -265,7 +284,7 @@ def bypass_captcha(captcha_key, logger)
           loop do
             sleep 10.seconds
 
-            response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
+            response = JSON.parse(captcha_client.post("/getTaskResult", body: {
               "clientKey" => CONFIG.captcha_key,
               "taskId"    => task_id,
             }.to_json).body)
@@ -283,7 +302,11 @@ def bypass_captcha(captcha_key, logger)
           yield response.cookies.select { |cookie| cookie.name != "PREF" }
         elsif response.headers["Location"]?.try &.includes?("/sorry/index")
           location = response.headers["Location"].try { |u| URI.parse(u) }
-          headers = HTTP::Headers{":authority" => location.host.not_nil!}
+          headers = HTTP::Headers{
+            ":authority" => location.host.not_nil!,
+            "origin"     => "https://www.google.com",
+            "user-agent" => "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36",
+          }
           response = YT_POOL.client &.get(location.full_path, headers)
 
           html = XML.parse_html(response.body)
@@ -297,14 +320,32 @@ def bypass_captcha(captcha_key, logger)
 
           captcha_client = HTTPClient.new(URI.parse("https://api.anti-captcha.com"))
           captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
-          response = JSON.parse(captcha_client.post("/createTask", body: {
+          if !CONFIG.proxy_address.empty?
-            "clientKey" => CONFIG.captcha_key,
+            response = JSON.parse(captcha_client.post("/createTask", body: {
-            "task"      => {
+              "clientKey" => CONFIG.captcha_key,
-              "type"       => "NoCaptchaTaskProxyless",
+              "task"      => {
-              "websiteURL" => location.to_s,
+                "type"          => "NoCaptchaTask",
-              "websiteKey" => site_key,
+                "websiteURL"    => location.to_s,
-            },
+                "websiteKey"    => site_key,
-          }.to_json).body)
+                "proxyType"     => "http",
+                "proxyAddress"  => CONFIG.proxy_address,
+                "proxyPort"     => CONFIG.proxy_port,
+                "proxyLogin"    => CONFIG.proxy_user,
+                "proxyPassword" => CONFIG.proxy_pass,
+                "userAgent"     => headers["user-agent"],
+              },
+            }.to_json).body)
+          else
+            response = JSON.parse(captcha_client.post("/createTask", body: {
+              "clientKey" => CONFIG.captcha_key,
+              "task"      => {
+                "type"       => "NoCaptchaTaskProxyless",
+                "websiteURL" => location.to_s,
+                "websiteKey" => site_key,
+                "userAgent"  => headers["user-agent"],
+              },
+            }.to_json).body)
+          end
 
           raise response["error"].as_s if response["error"]?
           task_id = response["taskId"].as_i
@@ -326,8 +367,7 @@ def bypass_captcha(captcha_key, logger)
 
           inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
           headers["content-type"] = "application/x-www-form-urlencoded"
-          headers["origin"] = "https://www.google.com"
+          headers["referer"] = location.to_s
-          headers["user-agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
 
           response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
           headers = HTTP::Headers{

src/invidious/helpers/proxy.cr

@@ -1,3 +1,7 @@
+def connect(path : String, &block : HTTP::Server::Context -> _)
+  Kemal::RouteHandler::INSTANCE.add_route("CONNECT", path, &block)
+end
+
 # See https://github.com/crystal-lang/crystal/issues/2963
 class HTTPProxy
   getter proxy_host : String
@@ -124,7 +128,7 @@ def get_nova_proxies(country_code = "US")
   client.connect_timeout = 10.seconds
 
   headers = HTTP::Headers.new
-  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
   headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
   headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
   headers["Host"] = "www.proxynova.com"
@@ -161,7 +165,7 @@ def get_spys_proxies(country_code = "US")
   client.connect_timeout = 10.seconds
 
   headers = HTTP::Headers.new
-  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
   headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
   headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
   headers["Host"] = "spys.one"

src/invidious/helpers/utils.cr

@@ -2,11 +2,12 @@ require "lsquic"
 require "pool/connection"
 
 def add_yt_headers(request)
-  request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
+  return if request.resource.starts_with? "/sorry/index"
+
+  request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
   request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
   request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
   request.headers["accept-language"] ||= "en-us,en;q=0.5"
-  return if request.resource.starts_with? "/sorry/index"
   request.headers["x-youtube-client-name"] ||= "1"
   request.headers["x-youtube-client-version"] ||= "1.20180719"
   if !CONFIG.cookies.empty?

src/invidious/mixes.cr

@@ -20,7 +20,7 @@ end
 
 def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
   headers = HTTP::Headers.new
-  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
+  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
 
   if cookies
     headers = cookies.add_request_headers(headers)

src/invidious/trending.cr

@@ -1,6 +1,6 @@
 def fetch_trending(trending_type, region, locale)
   headers = HTTP::Headers.new
-  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+  headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36"
 
   region ||= "US"
   region = region.upcase