This commit is contained in:
syeopite 2024-08-27 14:47:29 -07:00 committed by GitHub
commit da0a5de588
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 64 additions and 97 deletions

View File

@ -92,6 +92,10 @@ SOFTWARE = {
YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
# Image request pool
GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
# CLI # CLI
Kemal.config.extra_options do |parser| Kemal.config.extra_options do |parser|
parser.banner = "Usage: invidious [arguments]" parser.banner = "Usage: invidious [arguments]"

View File

@ -11,29 +11,9 @@ module Invidious::Routes::Images
end end
end end
# We're encapsulating this into a proc in order to easily reuse this
# portion of the code for each request block below.
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
env.response.headers.delete("Transfer-Encoding")
return
end
proxy_file(response, env)
}
begin begin
HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| GGPHT_POOL.client &.get(url, headers) do |resp|
return request_proc.call(resp) return self.proxy_image(env, resp)
end end
rescue ex rescue ex
end end
@ -61,27 +41,10 @@ module Invidious::Routes::Images
end end
end end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Connection"] = "close"
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin begin
HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| get_ytimg_pool(authority).client &.get(url, headers) do |resp|
return request_proc.call(resp) env.response.headers["Connection"] = "close"
return self.proxy_image(env, resp)
end end
rescue ex rescue ex
end end
@ -101,26 +64,9 @@ module Invidious::Routes::Images
end end
end end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin begin
HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| get_ytimg_pool("i9").client &.get(url, headers) do |resp|
return request_proc.call(resp) return self.proxy_image(env, resp)
end end
rescue ex rescue ex
end end
@ -165,8 +111,7 @@ module Invidious::Routes::Images
if name == "maxres.jpg" if name == "maxres.jpg"
build_thumbnails(id).each do |thumb| build_thumbnails(id).each do |thumb|
thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg"
# This can likely be optimized into a (small) pool sometime in the future. if get_ytimg_pool("i9").client &.head(thumbnail_resource_path, headers).status_code == 200
if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200
name = thumb[:url] + ".jpg" name = thumb[:url] + ".jpg"
break break
end end
@ -181,29 +126,28 @@ module Invidious::Routes::Images
end end
end end
request_proc = ->(response : HTTP::Client::Response) {
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300 && response.status_code != 404
return env.response.headers.delete("Transfer-Encoding")
end
proxy_file(response, env)
}
begin begin
# This can likely be optimized into a (small) pool sometime in the future. get_ytimg_pool("i").client &.get(url, headers) do |resp|
HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| return self.proxy_image(env, resp)
return request_proc.call(resp)
end end
rescue ex rescue ex
end end
end end
private def self.proxy_image(env, response)
env.response.status_code = response.status_code
response.headers.each do |key, value|
if !RESPONSE_HEADERS_BLACKLIST.includes?(key.downcase)
env.response.headers[key] = value
end
end
env.response.headers["Access-Control-Allow-Origin"] = "*"
if response.status_code >= 300
return env.response.headers.delete("Transfer-Encoding")
end
return proxy_file(response, env)
end
end end

View File

@ -1,17 +1,6 @@
def add_yt_headers(request) # Mapping of subdomain => YoutubeConnectionPool
request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal" # This is needed as we may need to access arbitrary subdomains of ytimg
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" private YTIMG_POOLS = {} of String => YoutubeConnectionPool
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
if !CONFIG.cookies.empty?
request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
struct YoutubeConnectionPool struct YoutubeConnectionPool
property! url : URI property! url : URI
@ -54,6 +43,21 @@ struct YoutubeConnectionPool
end end
end end
def add_yt_headers(request)
request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
# Preserve original cookies and add new YT consent cookie for EU servers
request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=PENDING+#{Random.rand(100..999)}"
if !CONFIG.cookies.empty?
request.headers["Cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
end
def make_client(url : URI, region = nil, force_resolve : Bool = false) def make_client(url : URI, region = nil, force_resolve : Bool = false)
client = HTTP::Client.new(url) client = HTTP::Client.new(url)
@ -77,3 +81,18 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
client.close client.close
end end
end end
# Fetches a HTTP pool for the specified subdomain of ytimg.com
#
# Creates a new one when the specified pool for the subdomain does not exist
def get_ytimg_pool(subdomain)
if pool = YTIMG_POOLS[subdomain]?
return pool
else
LOGGER.info("ytimg_pool: Creating a new HTTP pool for \"https://#{subdomain}.ytimg.com\"")
pool = YoutubeConnectionPool.new(URI.parse("https://#{subdomain}.ytimg.com"), capacity: CONFIG.pool_size)
YTIMG_POOLS[subdomain] = pool
return pool
end
end