Use new cache system for 'Video' objects

This commit is contained in:
Samantaz Fox 2023-06-18 13:10:39 +02:00
parent c847e357f9
commit d8dee8e767
No known key found for this signature in database
GPG Key ID: F42821059186176E
12 changed files with 58 additions and 61 deletions

View File

@ -18,7 +18,6 @@ module Invidious::Database
Invidious::Database.check_table("nonces", Nonce) Invidious::Database.check_table("nonces", Nonce)
Invidious::Database.check_table("session_ids", SessionId) Invidious::Database.check_table("session_ids", SessionId)
Invidious::Database.check_table("users", User) Invidious::Database.check_table("users", User)
Invidious::Database.check_table("videos", Video)
if cfg.cache_annotations if cfg.cache_annotations
Invidious::Database.check_table("annotations", Annotation) Invidious::Database.check_table("annotations", Annotation)

View File

@ -74,7 +74,7 @@ def create_notification_stream(env, topics, connection_channel)
published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
video_id = TEST_IDS[rand(TEST_IDS.size)] video_id = TEST_IDS[rand(TEST_IDS.size)]
video = get_video(video_id) video = Video.get(video_id)
video.published = published video.published = published
response = JSON.parse(video.to_json(locale, nil)) response = JSON.parse(video.to_json(locale, nil))
@ -133,7 +133,7 @@ def create_notification_stream(env, topics, connection_channel)
next next
end end
video = get_video(video_id) video = Video.get(video_id)
video.published = Time.unix(published) video.published = Time.unix(published)
response = JSON.parse(video.to_json(locale, nil)) response = JSON.parse(video.to_json(locale, nil))

View File

@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest
unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
begin begin
video = get_video(id, region: region) video = Video.get(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
haltf env, status_code: 404 haltf env, status_code: 404
rescue ex rescue ex

View File

@ -314,7 +314,7 @@ module Invidious::Routes::API::V1::Authenticated
end end
begin begin
video = get_video(video_id) video = Video.get(video_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return error_json(404, ex)
rescue ex rescue ex

View File

@ -9,7 +9,7 @@ module Invidious::Routes::API::V1::Videos
proxy = {"1", "true"}.any? &.== env.params.query["local"]? proxy = {"1", "true"}.any? &.== env.params.query["local"]?
begin begin
video = get_video(id, region: region) video = Video.get(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return error_json(404, ex)
rescue ex rescue ex
@ -40,7 +40,7 @@ module Invidious::Routes::API::V1::Videos
# getting video info. # getting video info.
begin begin
video = get_video(id, region: region) video = Video.get(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
haltf env, 404 haltf env, 404
rescue ex rescue ex
@ -180,7 +180,7 @@ module Invidious::Routes::API::V1::Videos
region = find_region(env.params.query["region"]?) region = find_region(env.params.query["region"]?)
begin begin
video = get_video(id, region: region) video = Video.get(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
haltf env, 404 haltf env, 404
rescue ex rescue ex

View File

@ -130,7 +130,7 @@ module Invidious::Routes::Embed
subscriptions ||= [] of String subscriptions ||= [] of String
begin begin
video = get_video(id, region: params.region) video = Video.get(id, region: params.region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return error_template(404, ex)
rescue ex rescue ex

View File

@ -420,7 +420,7 @@ module Invidious::Routes::Feeds
updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
begin begin
video = get_video(id, force_refresh: true) video = Video.get(id, force_refresh: true)
rescue rescue
next # skip this video since it raised an exception (e.g. it is a scheduled live event) next # skip this video since it raised an exception (e.g. it is a scheduled live event)
end end

View File

@ -352,7 +352,7 @@ module Invidious::Routes::Playlists
video_id = env.params.query["video_id"] video_id = env.params.query["video_id"]
begin begin
video = get_video(video_id) video = Video.get(video_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return error_json(404, ex)
rescue ex rescue ex

View File

@ -275,7 +275,7 @@ module Invidious::Routes::VideoPlayback
end end
begin begin
video = get_video(id, region: region) video = Video.get(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return error_template(404, ex)
rescue ex rescue ex

View File

@ -52,7 +52,7 @@ module Invidious::Routes::Watch
env.params.query.delete_all("listen") env.params.query.delete_all("listen")
begin begin
video = get_video(id, region: params.region) video = Video.get(id, region: params.region)
rescue ex : NotFoundException rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}") LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex) return error_template(404, ex)

View File

@ -59,7 +59,7 @@ struct Invidious::User
next if video_id == "Video Id" next if video_id == "Video Id"
begin begin
video = get_video(video_id) video = Video.get(video_id)
rescue ex rescue ex
next next
end end
@ -133,7 +133,7 @@ struct Invidious::User
next if !video_id next if !video_id
begin begin
video = get_video(video_id, false) video = Video.get(video_id)
rescue ex rescue ex
next next
end end

View File

@ -5,8 +5,6 @@ enum VideoType
end end
struct Video struct Video
include DB::Serializable
# Version of the JSON structure # Version of the JSON structure
# It prevents us from loading an incompatible version from cache # It prevents us from loading an incompatible version from cache
# (either newer or older, if instances with different versions run # (either newer or older, if instances with different versions run
@ -18,21 +16,13 @@ struct Video
SCHEMA_VERSION = 2 SCHEMA_VERSION = 2
property id : String property id : String
@[DB::Field(converter: Video::JSONConverter)]
property info : Hash(String, JSON::Any) property info : Hash(String, JSON::Any)
property updated : Time
@[DB::Field(ignore: true)]
@captions = [] of Invidious::Videos::Captions::Metadata @captions = [] of Invidious::Videos::Captions::Metadata
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))? property adaptive_fmts : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property fmt_stream : Array(Hash(String, JSON::Any))? property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String? property description : String?
module JSONConverter module JSONConverter
@ -41,6 +31,49 @@ struct Video
end end
end end
# Create new object from cache (JSON)
def initialize(@id, @info)
end
def self.get(id : String, *, force_refresh = false, region = nil)
key = "video:#{id}"
key += ":#{region}" if !region.nil?
# Fetch video from cache, unles a force refresh is requested
info = force_refresh ? nil : IV::Cache::INSTANCE.fetch(key)
updated = false
# Fetch video from youtube, if needed
if info.nil?
video = Video.new(id, fetch_video(id, region))
updated = true
else
video = Video.new(id, JSON.parse(info).as_h)
# If video has premiered, live has started or the format
# of the video data has changed, refresh the data.
outdated_data = (video.schema_version != Video::SCHEMA_VERSION)
live_started = (video.live_now && video.published < Time.utc)
if outdated_data || live_started
video = Video.new(id, fetch_video(id, region))
updated = true
end
end
# Store updated entry in cache
# TODO: finer cache control based on video type & publication date
if updated
if video.live_now || video.published < Time.utc
IV::Cache::INSTANCE.store(key, info.to_json, 10.minutes)
else
IV::Cache::INSTANCE.store(key, info.to_json, 2.hours)
end
end
return video
end
# Methods for API v1 JSON # Methods for API v1 JSON
def to_json(locale : String?, json : JSON::Builder) def to_json(locale : String?, json : JSON::Builder)
@ -362,35 +395,6 @@ struct Video
getset_bool isUpcoming getset_bool isUpcoming
end end
def get_video(id, refresh = true, region = nil, force_refresh = false)
if (video = Invidious::Database::Videos.select(id)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours)
if (refresh &&
(Time.utc - video.updated > 10.minutes) ||
(video.premiere_timestamp.try &.< Time.utc)) ||
force_refresh ||
video.schema_version != Video::SCHEMA_VERSION # cache control
begin
video = fetch_video(id, region)
Invidious::Database::Videos.update(video)
rescue ex
Invidious::Database::Videos.delete(id)
raise ex
end
end
else
video = fetch_video(id, region)
Invidious::Database::Videos.insert(video) if !region
end
return video
rescue DB::Error
# Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
# Note: All DB errors inherit from `DB::Error`
return fetch_video(id, region)
end
def fetch_video(id, region) def fetch_video(id, region)
info = extract_video_info(video_id: id) info = extract_video_info(video_id: id)
@ -408,13 +412,7 @@ def fetch_video(id, region)
end end
end end
video = Video.new({ return info
id: id,
info: info,
updated: Time.utc,
})
return video
end end
def process_continuation(query, plid, id) def process_continuation(query, plid, id)