From ee728092823d8e82f71f35c31da8a27efec0f1b5 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Sat, 26 Oct 2024 12:40:31 -0400 Subject: [PATCH 01/10] [Alternative] Fix for channel live videos --- src/invidious/channels/videos.cr | 66 +++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 19 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 6cc301424..e29d80ed5 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -23,29 +23,57 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene else 15 # Fallback to "videos" end - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end + if content_type == "livestreams" + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + else + sort_by_numerical = + case sort_by + when "newest" then 1_i64 + when "popular" then 2_i64 + when "oldest" then 4_i64 + else 1_i64 # Fallback to "newest" + end + end - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, + if content_type == "livestreams" + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, }, }, - }, - } + } + else + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, + }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "3:varint" => sort_by_numerical, + }, + }, + }, + } + end object_inner_1_encoded = object_inner_1 .try { |i| Protodec::Any.cast_json(i) } From c243d08afb8509f7a98cd7aa1b77d4f409a7a823 Mon Sep 17 00:00:00 2001 From: Brahim Hadriche Date: Wed, 30 Oct 2024 13:38:13 -0400 Subject: [PATCH 02/10] refactor --- src/invidious/channels/videos.cr | 49 +++++++++++++------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index e29d80ed5..bcdc8d8f8 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -23,6 +23,13 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene else 15 # Fallback to "videos" end + sort_type_numerical = + case content_type + when "videos" then 3 + when "livestreams" then 5 + else 3 # Fallback to "videos" + end + if content_type == "livestreams" sort_by_numerical = case sort_by @@ -41,39 +48,21 @@ def produce_channel_content_continuation(ucid, content_type, page = 1, auto_gene end end - if content_type == "livestreams" - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "5:varint" => sort_by_numerical, + object_inner_1 = { + "110:embedded" => { + "3:embedded" => { + "#{content_type_numerical}:embedded" => { + "1:embedded" => { + "1:string" => object_inner_2_encoded, }, + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "#{sort_type_numerical}:varint" => sort_by_numerical, }, }, - } - else - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "3:varint" => sort_by_numerical, - }, - }, - }, - } - end + }, + } object_inner_1_encoded = object_inner_1 .try { |i| Protodec::Any.cast_json(i) } From cbc546f0320e4833927a654c26d384bb2e8a9f93 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Nov 2024 22:54:21 +0100 Subject: [PATCH 03/10] Channels: Add function to generate the new ctoken objects --- src/invidious/channels/videos.cr | 207 +++++++++++++++---------------- 1 file changed, 103 insertions(+), 104 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index bcdc8d8f8..7b3e3cfa1 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -1,95 +1,3 @@ -def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false) - object_inner_2 = { - "2:0:embedded" => { - "1:0:varint" => 0_i64, - }, - "5:varint" => 50_i64, - "6:varint" => 1_i64, - "7:varint" => (page * 30).to_i64, - "9:varint" => 1_i64, - "10:varint" => 0_i64, - } - - object_inner_2_encoded = object_inner_2 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - content_type_numerical = - case content_type - when "videos" then 15 - when "livestreams" then 14 - else 15 # Fallback to "videos" - end - - sort_type_numerical = - case content_type - when "videos" then 3 - when "livestreams" then 5 - else 3 # Fallback to "videos" - end - - if content_type == "livestreams" - sort_by_numerical = - case sort_by - when "newest" then 12_i64 - when "popular" then 14_i64 - when "oldest" then 13_i64 - else 12_i64 # Fallback to "newest" - end - else - sort_by_numerical = - case sort_by - when "newest" then 1_i64 - when "popular" then 2_i64 - when "oldest" then 4_i64 - else 1_i64 # Fallback to "newest" - end - end - - object_inner_1 = { - "110:embedded" => { - "3:embedded" => { - "#{content_type_numerical}:embedded" => { - "1:embedded" => { - "1:string" => object_inner_2_encoded, - }, - "2:embedded" => { - "1:string" => "00000000-0000-0000-0000-000000000000", - }, - "#{sort_type_numerical}:varint" => sort_by_numerical, - }, - }, - }, - } - - object_inner_1_encoded = object_inner_1 - .try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - object = { - "80226972:embedded" => { - "2:string" => ucid, - "3:string" => object_inner_1_encoded, - "35:string" => "browse-feed#{ucid}videos102", - }, - } - - continuation = object.try { |i| Protodec::Any.cast_json(i) } - .try { |i| Protodec::Any.from_json(i) } - .try { |i| Base64.urlsafe_encode(i) } - .try { |i| URI.encode_www_form(i) } - - return continuation -end - -def make_initial_content_ctoken(ucid, content_type, sort_by) : String - return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by) -end - module Invidious::Channel::Tabs extend self @@ -118,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by) + continuation ||= make_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -147,14 +55,10 @@ module Invidious::Channel::Tabs # Shorts # ------------------- - def get_shorts(channel : AboutChannel, continuation : String? = nil) - if continuation.nil? - # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts" - # TODO: try to extract the continuation tokens that allows other sorting options - initial_data = YoutubeAPI.browse(channel.ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D") - else - initial_data = YoutubeAPI.browse(continuation: continuation) - end + def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_shorts_ctoken(channel.ucid, sort_by) + initial_data = YoutubeAPI.browse(continuation: continuation) + return extract_items(initial_data, channel.author, channel.ucid) end @@ -162,9 +66,8 @@ module Invidious::Channel::Tabs # Livestreams # ------------------- - def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest") - continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by) - + def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") + continuation ||= make_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -188,4 +91,100 @@ module Invidious::Channel::Tabs return items, next_continuation end + + # ------------------- + # C-tokens + # ------------------- + + private def sort_options_videos_short(sort_by : String) + case sort_by + when "newest" then return 4_i64 + when "popular" then return 2_i64 + when "oldest" then return 5_i64 + else return 4_i64 # Fallback to "newest" + end + end + + # Generate the initial "continuation token" to get the first page of the + # "videos" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_videos_ctoken(ucid : String, sort_by = "newest") + object = { + "15:embedded" => { + "2:string" => "\n$00000000-0000-0000-0000-000000000000", + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "shorts" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_shorts_ctoken(ucid : String, sort_by = "newest") + object = { + "10:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "4:varint" => sort_options_videos_short(sort_by), + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # Generate the initial "continuation token" to get the first page of the + # "livestreams" tab. The following page requires the ctoken provided in that + # first page, and so on. + private def make_livestreams_ctoken(ucid : String, sort_by = "newest") + sort_by_numerical = + case sort_by + when "newest" then 12_i64 + when "popular" then 14_i64 + when "oldest" then 13_i64 + else 12_i64 # Fallback to "newest" + end + + object = { + "14:embedded" => { + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, + "5:varint" => sort_by_numerical, + }, + } + + return channel_ctoken_wrap(ucid, object) + end + + # The protobuf structure common between videos/shorts/livestreams + private def channel_ctoken_wrap(ucid : String, object) + object_inner = { + "110:embedded" => { + "3:embedded" => object, + }, + } + + object_inner_encoded = object_inner + .try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + object = { + "80226972:embedded" => { + "2:string" => ucid, + "3:string" => object_inner_encoded, + }, + } + + continuation = object.try { |i| Protodec::Any.cast_json(i) } + .try { |i| Protodec::Any.from_json(i) } + .try { |i| Base64.urlsafe_encode(i) } + .try { |i| URI.encode_www_form(i) } + + return continuation + end end From 82248fad024de5289011e2ae26d5c390d5084827 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Thu, 7 Nov 2024 23:00:18 +0100 Subject: [PATCH 04/10] Channels: Add sort options to shorts --- src/invidious/routes/channels.cr | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 952098e0a..d4e9fa681 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -82,13 +82,12 @@ module Invidious::Routes::Channels end next_continuation = nil else - # TODO: support sort option for shorts - sort_by = "" - sort_options = [] of String + sort_by = env.params.query["sort_by"]?.try &.downcase || "newest" + sort_options = {"newest", "oldest", "popular"} # Fetch items and continuation token items, next_continuation = Channel::Tabs.get_shorts( - channel, continuation: continuation + channel, continuation: continuation, sort_by: sort_by ) end From 1a5047aad94454fd8a8d9623e17ee3782c68c3d0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 12:33:14 +0100 Subject: [PATCH 05/10] Extractors: Add support for lockupViewModel The 'lockupViewModel' structure is used in the channel "podcasts" tab --- src/invidious/yt_backend/extractors.cr | 76 +++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4074de860..cb8331a56 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -467,9 +467,9 @@ private module Parsers # Parses an InnerTube richItemRenderer into a SearchVideo. # Returns nil when the given object isn't a RichItemRenderer # - # A richItemRenderer seems to be a simple wrapper for a videoRenderer, used - # by the result page for hashtags and for the podcast tab on channels. - # It is located inside a continuationItems container for hashtags. + # A richItemRenderer seems to be a simple wrapper for a various other types, + # used on the hashtags result page and the channel podcast tab. It is located + # itself inside a richGridRenderer container. # module RichItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -482,6 +482,7 @@ private module Parsers child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) + child ||= LockupViewModelParser.process(item_contents, author_fallback) return child end @@ -582,6 +583,75 @@ private module Parsers end end + # Parses an InnerTube lockupViewModel into a SearchPlaylist. + # Returns nil when the given object is not a lockupViewModel. + # + # This structure is present since November 2024 on the "podcasts" tab of the + # channel page. It is usually (always?) encapsulated in a richItemRenderer. + # + module LockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["lockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + playlist_id = item_contents["contentId"].as_s + + thumbnail_view_model = item_contents.dig( + "contentImage", "collectionThumbnailViewModel", + "primaryThumbnail", "thumbnailViewModel" + ) + + thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s + + # This complicated sequences tries to extract the following data structure: + # "overlays": [{ + # "thumbnailOverlayBadgeViewModel": { + # "thumbnailBadges": [{ + # "thumbnailBadgeViewModel": { + # "text": "430 episodes", + # "badgeStyle": "THUMBNAIL_OVERLAY_BADGE_STYLE_DEFAULT" + # } + # }] + # } + # }] + video_count = thumbnail_view_model.dig("overlays").as_a + .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) + .flatten + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes")) + .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) + + metadata = item_contents.dig("metadata", "lockupMetadataViewModel") + title = metadata.dig("title", "content").as_s + + # TODO: Retrieve "updated" info from metadata parts + # rows = metadata.dig("metadata", "contentMetadataViewModel", "metadataRows").as_a + # parts_text = rows.map(&.dig?("metadataParts", "text", "content").try &.as_s) + # One of these parts should contain a string like: "Updated 2 days ago" + + # TODO: Maybe add a button to access the first video of the playlist? + # item_contents.dig("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint") + # Available fields: "videoId", "playlistId", "params" + + return SearchPlaylist.new({ + title: title, + id: playlist_id, + author: author_fallback.name, + ucid: author_fallback.id, + video_count: video_count || -1, + videos: [] of SearchPlaylistVideo, + thumbnail: thumbnail, + author_verified: false, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube continuationItemRenderer into a Continuation. # Returns nil when the given object isn't a continuationItemRenderer. # From afc5b27d83d8b2b287842ed1ec43185135441d37 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:32:44 +0100 Subject: [PATCH 06/10] Extractors: Add support for shortsLockupViewModel The 'shortsLockupViewModel' structure is used in the channel "shorts" tab --- src/invidious/yt_backend/extractors.cr | 58 ++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index cb8331a56..4416ef30e 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -483,6 +483,7 @@ private module Parsers child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) child ||= LockupViewModelParser.process(item_contents, author_fallback) + child ||= ShortsLockupViewModelParser.process(item_contents, author_fallback) return child end @@ -497,6 +498,9 @@ private module Parsers # reelItemRenderer items are used in the new (2022) channel layout, # in the "shorts" tab. # + # NOTE: As of 10/2024, it might have been fully replaced by shortsLockupViewModel + # TODO: Confirm that hypothesis + # module ReelItemRendererParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? @@ -652,6 +656,60 @@ private module Parsers end end + # Parses an InnerTube shortsLockupViewModel into a SearchVideo. + # Returns nil when the given object is not a shortsLockupViewModel. + # + # This structure is present since around October 2024 on the "shorts" tab of + # the channel page and likely replaces the reelItemRenderer structure. It is + # usually (always?) encapsulated in a richItemRenderer. + # + module ShortsLockupViewModelParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) + if item_contents = item["shortsLockupViewModel"]? + return self.parse(item_contents, author_fallback) + end + end + + private def self.parse(item_contents, author_fallback) + # TODO: Maybe add support for "oardefault.jpg" thumbnails? + # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s + # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... + + video_id = item_contents.dig( + "onTap", "innertubeCommand", "reelWatchEndpoint", "videoId" + ).as_s + + title = item_contents.dig("overlayMetadata", "primaryText", "content").as_s + + view_count = short_text_to_number( + item_contents.dig("overlayMetadata", "secondaryText", "content").as_s + ) + + # Approximate to one minute, as "shorts" generally don't exceed that. + # NOTE: The actual duration is not provided by Youtube anymore. + # TODO: Maybe use -1 as an error value and handle that on the frontend? + duration = 60_i32 + + SearchVideo.new({ + title: title, + id: video_id, + author: author_fallback.name, + ucid: author_fallback.id, + published: Time.unix(0), + views: view_count, + description_html: "", + length_seconds: duration, + premiere_timestamp: Time.unix(0), + author_verified: false, + badges: VideoBadges::None, + }) + end + + def self.parser_name + return {{@type.name}} + end + end + # Parses an InnerTube continuationItemRenderer into a Continuation. # Returns nil when the given object isn't a continuationItemRenderer. # From d27a5e7fae4a826b66950422ff8dfec4123dabf1 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:33:46 +0100 Subject: [PATCH 07/10] Channels: Rename ctoken generator functions as requested --- src/invidious/channels/videos.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 7b3e3cfa1..9572adf32 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -26,7 +26,7 @@ module Invidious::Channel::Tabs end def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_videos_ctoken(ucid, sort_by) + continuation ||= make_initial_videos_ctoken(ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, author, ucid) @@ -56,7 +56,7 @@ module Invidious::Channel::Tabs # ------------------- def get_shorts(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_shorts_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_shorts_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -67,7 +67,7 @@ module Invidious::Channel::Tabs # ------------------- def get_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest") - continuation ||= make_livestreams_ctoken(channel.ucid, sort_by) + continuation ||= make_initial_livestreams_ctoken(channel.ucid, sort_by) initial_data = YoutubeAPI.browse(continuation: continuation) return extract_items(initial_data, channel.author, channel.ucid) @@ -108,7 +108,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "videos" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_videos_ctoken(ucid : String, sort_by = "newest") + private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") object = { "15:embedded" => { "2:string" => "\n$00000000-0000-0000-0000-000000000000", @@ -122,7 +122,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "shorts" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_shorts_ctoken(ucid : String, sort_by = "newest") + private def make_initial_shorts_ctoken(ucid : String, sort_by = "newest") object = { "10:embedded" => { "2:embedded" => { @@ -138,7 +138,7 @@ module Invidious::Channel::Tabs # Generate the initial "continuation token" to get the first page of the # "livestreams" tab. The following page requires the ctoken provided in that # first page, and so on. - private def make_livestreams_ctoken(ucid : String, sort_by = "newest") + private def make_initial_livestreams_ctoken(ucid : String, sort_by = "newest") sort_by_numerical = case sort_by when "newest" then 12_i64 From 301aeffa780fca321793f8c2ef46844d613ce5c3 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 13:54:05 +0100 Subject: [PATCH 08/10] Channels: Multiple small fixes Fix the "newest" link not being bold when 'sort_by' uses the default value Show 60 videos per page, rather than 30 --- src/invidious/routes/channels.cr | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index d4e9fa681..7d634cbb6 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -20,10 +20,11 @@ module Invidious::Routes::Channels sort_by = env.params.query["sort_by"]?.try &.downcase if channel.auto_generated + sort_by ||= "last" sort_options = {"last", "oldest", "newest"} items, next_continuation = fetch_channel_playlists( - channel.ucid, channel.author, continuation, (sort_by || "last") + channel.ucid, channel.author, continuation, sort_by ) items.uniq! do |item| @@ -49,9 +50,11 @@ module Invidious::Routes::Channels end next_continuation = nil else + sort_by ||= "newest" sort_options = {"newest", "oldest", "popular"} - items, next_continuation = Channel::Tabs.get_videos( - channel, continuation: continuation, sort_by: (sort_by || "newest") + + items, next_continuation = Channel::Tabs.get_60_videos( + channel, continuation: continuation, sort_by: sort_by ) end end From 6dd662a5b84b3deb9e19e365f8b480357f63a2e9 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 17:25:23 +0100 Subject: [PATCH 09/10] Channels: lockupViewModel is also used in the "playlists" tab --- src/invidious/yt_backend/extractors.cr | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 4416ef30e..2631b62a5 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -21,6 +21,7 @@ private ITEM_PARSERS = { Parsers::ItemSectionRendererParser, Parsers::ContinuationItemRendererParser, Parsers::HashtagRendererParser, + Parsers::LockupViewModelParser, } private alias InitialData = Hash(String, JSON::Any) @@ -590,8 +591,9 @@ private module Parsers # Parses an InnerTube lockupViewModel into a SearchPlaylist. # Returns nil when the given object is not a lockupViewModel. # - # This structure is present since November 2024 on the "podcasts" tab of the - # channel page. It is usually (always?) encapsulated in a richItemRenderer. + # This structure is present since November 2024 on the "podcasts" and + # "playlists" tabs of the channel page. It is usually encapsulated in either + # a richItemRenderer or a richGridRenderer. # module LockupViewModelParser def self.process(item : JSON::Any, author_fallback : AuthorFallback) @@ -608,7 +610,7 @@ private module Parsers "primaryThumbnail", "thumbnailViewModel" ) - thumbnail = thumbnail_view_model.dig("image", "sources", 1, "url").as_s + thumbnail = thumbnail_view_model.dig("image", "sources", 0, "url").as_s # This complicated sequences tries to extract the following data structure: # "overlays": [{ @@ -621,10 +623,15 @@ private module Parsers # }] # } # }] + # + # NOTE: this simplistic `.to_i` conversion might not work on larger + # playlists and hasn't been tested. video_count = thumbnail_view_model.dig("overlays").as_a .compact_map(&.dig?("thumbnailOverlayBadgeViewModel", "thumbnailBadges").try &.as_a) .flatten - .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try &.as_s.ends_with?("episodes")) + .find(nil, &.dig?("thumbnailBadgeViewModel", "text").try { |node| + {"episodes", "videos"}.any? { |str| node.as_s.ends_with?(str) } + }) .try &.dig("thumbnailBadgeViewModel", "text").as_s.to_i(strict: false) metadata = item_contents.dig("metadata", "lockupMetadataViewModel") From 2a19dbb1fee20e5438751c3bb387f8757f4c2238 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Fri, 8 Nov 2024 18:28:55 +0100 Subject: [PATCH 10/10] Channels: Use the same structure as in the other ctoken functions Change explanation, courtesy of iBicha: The \n is basically a decimal 10, which is 1010 binary. That is a field number 1, and a wire type 2 (length-delimited). Then the $ is a decimal 36, which is exactly the length of 00000000-0000-0000-0000-000000000000. So both objects end up being encoded into the same data. --- src/invidious/channels/videos.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 9572adf32..96400f471 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -111,7 +111,9 @@ module Invidious::Channel::Tabs private def make_initial_videos_ctoken(ucid : String, sort_by = "newest") object = { "15:embedded" => { - "2:string" => "\n$00000000-0000-0000-0000-000000000000", + "2:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + }, "4:varint" => sort_options_videos_short(sort_by), }, }