diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr index 0fb1b6af..3a6a15c6 100644 --- a/src/invidious/database/base.cr +++ b/src/invidious/database/base.cr @@ -18,7 +18,6 @@ module Invidious::Database Invidious::Database.check_table("nonces", Nonce) Invidious::Database.check_table("session_ids", SessionId) Invidious::Database.check_table("users", User) - Invidious::Database.check_table("videos", Video) if cfg.cache_annotations Invidious::Database.check_table("annotations", Annotation) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6add0237..80aa092b 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -74,7 +74,7 @@ def create_notification_stream(env, topics, connection_channel) published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) video_id = TEST_IDS[rand(TEST_IDS.size)] - video = get_video(video_id) + video = Video.get(video_id) video.published = published response = JSON.parse(video.to_json(locale, nil)) @@ -133,7 +133,7 @@ def create_notification_stream(env, topics, connection_channel) next end - video = get_video(video_id) + video = Video.get(video_id) video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) diff --git a/src/invidious/routes/api/manifest.cr b/src/invidious/routes/api/manifest.cr index 1a851a9b..a93160a8 100644 --- a/src/invidious/routes/api/manifest.cr +++ b/src/invidious/routes/api/manifest.cr @@ -13,7 +13,7 @@ module Invidious::Routes::API::Manifest unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe } begin - video = get_video(id, region: region) + video = Video.get(id, region: region) rescue ex : NotFoundException haltf env, status_code: 404 rescue ex diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index a35d2f2b..05040174 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -314,7 +314,7 @@ module Invidious::Routes::API::V1::Authenticated end begin - video = get_video(video_id) + video = Video.get(video_id) rescue ex : NotFoundException return error_json(404, ex) rescue ex diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 47d2ae5d..069ccc6d 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -9,7 +9,7 @@ module Invidious::Routes::API::V1::Videos proxy = {"1", "true"}.any? &.== env.params.query["local"]? begin - video = get_video(id, region: region) + video = Video.get(id, region: region) rescue ex : NotFoundException return error_json(404, ex) rescue ex @@ -40,7 +40,7 @@ module Invidious::Routes::API::V1::Videos # getting video info. begin - video = get_video(id, region: region) + video = Video.get(id, region: region) rescue ex : NotFoundException haltf env, 404 rescue ex @@ -180,7 +180,7 @@ module Invidious::Routes::API::V1::Videos region = find_region(env.params.query["region"]?) begin - video = get_video(id, region: region) + video = Video.get(id, region: region) rescue ex : NotFoundException haltf env, 404 rescue ex diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 266f7ba4..2d218f17 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -130,7 +130,7 @@ module Invidious::Routes::Embed subscriptions ||= [] of String begin - video = get_video(id, region: params.region) + video = Video.get(id, region: params.region) rescue ex : NotFoundException return error_template(404, ex) rescue ex diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 04bccf81..02d2b37d 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -420,7 +420,7 @@ module Invidious::Routes::Feeds updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content) begin - video = get_video(id, force_refresh: true) + video = Video.get(id, force_refresh: true) rescue next # skip this video since it raised an exception (e.g. it is a scheduled live event) end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index abd0e945..968724a5 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -352,7 +352,7 @@ module Invidious::Routes::Playlists video_id = env.params.query["video_id"] begin - video = get_video(video_id) + video = Video.get(video_id) rescue ex : NotFoundException return error_json(404, ex) rescue ex diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 74ad1d1d..127a80bc 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -275,7 +275,7 @@ module Invidious::Routes::VideoPlayback end begin - video = get_video(id, region: region) + video = Video.get(id, region: region) rescue ex : NotFoundException return error_template(404, ex) rescue ex diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 410e625b..3beb9ec2 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -52,7 +52,7 @@ module Invidious::Routes::Watch env.params.query.delete_all("listen") begin - video = get_video(id, region: params.region) + video = Video.get(id, region: params.region) rescue ex : NotFoundException LOGGER.error("get_video not found: #{id} : #{ex.message}") return error_template(404, ex) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 108f2ccc..76fb6ba9 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -59,7 +59,7 @@ struct Invidious::User next if video_id == "Video Id" begin - video = get_video(video_id) + video = Video.get(video_id) rescue ex next end @@ -133,7 +133,7 @@ struct Invidious::User next if !video_id begin - video = get_video(video_id, false) + video = Video.get(video_id) rescue ex next end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index cdfca02c..d5f1ced8 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -5,8 +5,6 @@ enum VideoType end struct Video - include DB::Serializable - # Version of the JSON structure # It prevents us from loading an incompatible version from cache # (either newer or older, if instances with different versions run @@ -18,21 +16,13 @@ struct Video SCHEMA_VERSION = 2 property id : String - - @[DB::Field(converter: Video::JSONConverter)] property info : Hash(String, JSON::Any) - property updated : Time - @[DB::Field(ignore: true)] @captions = [] of Invidious::Videos::Captions::Metadata - @[DB::Field(ignore: true)] property adaptive_fmts : Array(Hash(String, JSON::Any))? - - @[DB::Field(ignore: true)] property fmt_stream : Array(Hash(String, JSON::Any))? - @[DB::Field(ignore: true)] property description : String? module JSONConverter @@ -41,6 +31,49 @@ struct Video end end + # Create new object from cache (JSON) + def initialize(@id, @info) + end + + def self.get(id : String, *, force_refresh = false, region = nil) + key = "video:#{id}" + key += ":#{region}" if !region.nil? + + # Fetch video from cache, unles a force refresh is requested + info = force_refresh ? nil : IV::Cache::INSTANCE.fetch(key) + updated = false + + # Fetch video from youtube, if needed + if info.nil? + video = Video.new(id, fetch_video(id, region)) + updated = true + else + video = Video.new(id, JSON.parse(info).as_h) + + # If video has premiered, live has started or the format + # of the video data has changed, refresh the data. + outdated_data = (video.schema_version != Video::SCHEMA_VERSION) + live_started = (video.live_now && video.published < Time.utc) + + if outdated_data || live_started + video = Video.new(id, fetch_video(id, region)) + updated = true + end + end + + # Store updated entry in cache + # TODO: finer cache control based on video type & publication date + if updated + if video.live_now || video.published < Time.utc + IV::Cache::INSTANCE.store(key, info.to_json, 10.minutes) + else + IV::Cache::INSTANCE.store(key, info.to_json, 2.hours) + end + end + + return video + end + # Methods for API v1 JSON def to_json(locale : String?, json : JSON::Builder) @@ -362,35 +395,6 @@ struct Video getset_bool isUpcoming end -def get_video(id, refresh = true, region = nil, force_refresh = false) - if (video = Invidious::Database::Videos.select(id)) && !region - # If record was last updated over 10 minutes ago, or video has since premiered, - # refresh (expire param in response lasts for 6 hours) - if (refresh && - (Time.utc - video.updated > 10.minutes) || - (video.premiere_timestamp.try &.< Time.utc)) || - force_refresh || - video.schema_version != Video::SCHEMA_VERSION # cache control - begin - video = fetch_video(id, region) - Invidious::Database::Videos.update(video) - rescue ex - Invidious::Database::Videos.delete(id) - raise ex - end - end - else - video = fetch_video(id, region) - Invidious::Database::Videos.insert(video) if !region - end - - return video -rescue DB::Error - # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends - # Note: All DB errors inherit from `DB::Error` - return fetch_video(id, region) -end - def fetch_video(id, region) info = extract_video_info(video_id: id) @@ -408,13 +412,7 @@ def fetch_video(id, region) end end - video = Video.new({ - id: id, - info: info, - updated: Time.utc, - }) - - return video + return info end def process_continuation(query, plid, id)