From 1b569bbc99207cae7c20aa285f42477ae361dd30 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 8 May 2021 21:09:30 -0700 Subject: [PATCH] Add support channel home pages + gen. improvements This commit adds support for channel home pages and all of the categories within it. However, the frontend code is a mess and thus needs to be refactor soon. Though that would likely require a rework of items.ecr This commit also comes with some general cleanups and improvements. Before this commit channel brand URls would only be supported on the videos page (now home page). It has been improved to be able to handle all channel URLs. The category_type and auxiliary_data property has also been removed from the Category struct. The former was never used and the latter allows for random data to be added to the Struct presenting documentation issues. Since the auxiliary_data variable was mainly used to store values from the browse_endpoint in order to create URLs, its much simpler to instead just get the URL from the webCommandMetadata. As a result of this change the browse_endpoint_data attribute of Category has also been removed. --- assets/css/channel.css | 21 ++++++- assets/css/default.css | 8 ++- src/invidious.cr | 20 ++----- src/invidious/channels.cr | 42 +++++++++---- src/invidious/helpers/extractors.cr | 38 ++---------- src/invidious/helpers/invidiousitems.cr | 7 +-- src/invidious/routes/channels.cr | 42 ++++++++++++- src/invidious/videos.cr | 2 +- src/invidious/views/channel/channel.ecr | 2 +- src/invidious/views/channel/community.ecr | 2 +- .../views/channel/featured_channels.ecr | 7 +-- src/invidious/views/channel/home.ecr | 60 +++++++++++++++++++ src/invidious/views/channel/playlists.ecr | 2 +- .../views/components/channel-information.ecr | 26 ++++++-- 14 files changed, 197 insertions(+), 82 deletions(-) diff --git a/assets/css/channel.css b/assets/css/channel.css index 85d66a685..c4f031343 100644 --- a/assets/css/channel.css +++ b/assets/css/channel.css @@ -66,7 +66,6 @@ } .category-heading { - font-size: 1.2em; user-select: none; display: inline; } @@ -117,3 +116,23 @@ only show up when the screen is wide enough */ margin-top: 1em; } } + +.trailer-metadata { + margin-left: 15px; + font-size: 12px; + color: rgb(232, 230, 227); +} + +.trailer-metadata .read-more { + line-height: 20px; + text-transform: uppercase; + color: gray; + font-size: 13px; +} + +.trailer-description { + overflow: hidden; + max-height: 150px; + line-height: 20px; + margin-top: 20px; +} \ No newline at end of file diff --git a/assets/css/default.css b/assets/css/default.css index 3434404e7..25e4262f5 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -602,7 +602,8 @@ hr { } .category { - margin: 3em 0px 4em 0px; + margin-bottom: 2em; + margin-top: 1em; } .category .heading > p { @@ -616,4 +617,9 @@ hr { border-radius: 5px; font-size: 14px; margin-left: 10px; +} + + /* Temp */ +.category-description { + color: #A8A095; } \ No newline at end of file diff --git a/src/invidious.cr b/src/invidious.cr index 0977f3a71..375d2e93e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -315,6 +315,10 @@ Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, Invidious::Routing.get "/channel/:ucid/channels", Invidious::Routes::Channels, :channels Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about +["", "/home", "/videos", "/playlists", "/community", "/channels", "/about"].each do |path| + Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect +end + Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect @@ -1624,22 +1628,6 @@ end end end -# YouTube appears to let users set a "brand" URL that -# is different from their username, so we convert that here -get "/c/:user" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.params.url["user"] - - response = YT_POOL.client &.get("/c/#{user}") - html = XML.parse_html(response.body) - - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next env.redirect "/" if !ucid - - env.redirect "/channel/#{ucid}" -end - # Legacy endpoint for /user/:username get "/profile" do |env| user = env.params.query["user"]? diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index bdfe5aa60..7e0b1353a 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -341,6 +341,30 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) return channel end +def fetch_channel_home(ucid, channel) + initial_data = request_youtube_api_browse(ucid, channel.tabs["home"][1]) + items = extract_items(initial_data, channel.author, channel.ucid) + + # Channel trailer needs some slight special handling + home_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) + trailer = home_tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["channelVideoPlayerRenderer"]? || nil + + home_sections = [] of (Category | Video) + if trailer + trailer = get_video(trailer["videoId"].as_s, PG_DB) + home_sections << trailer + end + + items.each do |category| + if category.is_a? Category + home_sections << category + end + end + + return home_sections + +end + def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = request_youtube_api_browse(continuation) @@ -381,8 +405,6 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) end def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, continuation = nil, query_title = nil) : {Array(Category), (String | Nil)} - auxiliary_data = {} of String => String - if continuation.is_a?(String) initial_data = request_youtube_api_browse(continuation) items = extract_items(initial_data) @@ -392,14 +414,13 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, title: query_title.not_nil!, # If continuation contents is requested then the query_title has to be passed along. contents: items, description_html: "", - browse_endpoint_data: nil, + url: nil, badges: nil, - auxiliary_data: auxiliary_data, })], continuation_token else + url = nil if view && shelf_id - auxiliary_data["view"] = view - auxiliary_data["shelf_id"] = shelf_id + url = "/channel/#{ucid}/channels?view=#{view}&shelf_id=#{shelf_id}" params = produce_featured_channel_browse_param(view.to_i64, shelf_id.to_i64) initial_data = request_youtube_api_browse(ucid, params) @@ -437,21 +458,20 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, title: category.title.empty? ? fallback_title : category.title, contents: category.contents, description_html: category.description_html, - browse_endpoint_data: nil, + url: category.url, badges: nil, - auxiliary_data: category.auxiliary_data, }) end - # If we don't have any categories we'll create one. + # If no categories has been parsed then it means that we're currently requesting a single one and not in + # the initial preview anymore. The frontend still needs a Category however, so we'll create one. if category_array.empty? category_array << Category.new({ title: fallback_title, contents: items, description_html: "", - browse_endpoint_data: nil, + url: url, badges: nil, - auxiliary_data: auxiliary_data, }) end diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index 430bb41c0..9da0bd6b8 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -219,37 +219,7 @@ private class CategoryParser < ItemParser title = "" end - auxiliary_data = {} of String => String - browse_endpoint = item_contents["endpoint"]?.try &.["browseEndpoint"] || nil - browse_endpoint_data = "" - category_type = 0 # 0: Video, 1: Channels, 2: Playlist/feed, 3: trending - - # There's no endpoint data for video and trending category - if !item_contents["endpoint"]? - if !item_contents["videoId"]? - category_type = 3 - end - end - - if !browse_endpoint.nil? - # Playlist/feed categories doesn't need the params value (nor is it even included in yt response) - # instead it uses the browseId parameter. So if there isn't a params value we can assume the - # category is a playlist/feed - if browse_endpoint["params"]? - # However, even though the channel category type returns the browse endpoint param - # we're not going to be using it in order to preserve compatablity with Youtube. - # and for an URL that looks cleaner - url = item_contents["endpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - url = URI.parse(url.as_s) - auxiliary_data["view"] = url.query_params["view"] - auxiliary_data["shelf_id"] = url.query_params["shelf_id"] - - category_type = 1 - else - browse_endpoint_data = browse_endpoint["browseId"].as_s - category_type = 2 - end - end + url = item_contents["endpoint"]?.try &.["commandMetadata"]["webCommandMetadata"]["url"].as_s # Sometimes a category can have badges. badges = [] of Tuple(String, String) # (Badge style, label) @@ -284,9 +254,8 @@ private class CategoryParser < ItemParser title: title, contents: contents, description_html: description_html, - browse_endpoint_data: browse_endpoint_data, + url: url, badges: badges, - auxiliary_data: auxiliary_data, }) end end @@ -325,6 +294,9 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor raw_items << renderer_container_contents next elsif items_container = renderer_container_contents["gridRenderer"]? + elsif items_container = renderer_container_contents["channelVideoPlayerRenderer"]? + # Parsing for channel trailer is already taken elsewhere + next else items_container = renderer_container_contents end diff --git a/src/invidious/helpers/invidiousitems.cr b/src/invidious/helpers/invidiousitems.cr index 83bd53205..9bb7e8677 100644 --- a/src/invidious/helpers/invidiousitems.cr +++ b/src/invidious/helpers/invidiousitems.cr @@ -232,14 +232,11 @@ class Category include DB::Serializable property title : String - property contents : Array(SearchItem) - property browse_endpoint_data : String? + property contents : Array(SearchItem) | Array(Video) + property url : String? property description_html : String property badges : Array(Tuple(String, String))? - # Data unique to only specific types of categories. - property auxiliary_data : Hash(String, String) - def to_json(locale, json : JSON::Builder) json.object do json.field "title", self.title diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index e6e402345..7cd83630b 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,18 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute def home(env) - self.videos(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + items = fetch_channel_home(ucid, channel) + + has_trailer = false + if items[0].is_a? Video + has_trailer = true + end + + templated "channel/home" end def videos(env) @@ -149,6 +161,34 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute templated "channel/about", buffer_footer: true end + def brand_redirect(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.params.url["user"] + + response = YT_POOL.client &.get("/c/#{user}") + html = XML.parse_html(response.body) + + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + if !ucid + env.response.status_code = 404 + return + end + + url = "/channel/#{ucid}" + + location = env.request.path.lchop?("/c/#{user}/") + if location + url += "/#{location}" + end + + if env.params.query.size > 0 + url += "?#{env.params.query}" + end + + env.redirect url + end + private def fetch_basic_information(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 116aafc77..a124dabe0 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -275,7 +275,7 @@ struct Video end end - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) json.object do json.field "type", "video" diff --git a/src/invidious/views/channel/channel.ecr b/src/invidious/views/channel/channel.ecr index c0caa80ac..1396e3971 100644 --- a/src/invidious/views/channel/channel.ecr +++ b/src/invidious/views/channel/channel.ecr @@ -4,7 +4,7 @@ <% end %> -<% content_type = 0 %> +<% content_type = 1 %> <%= rendered "components/channel-information" %>
diff --git a/src/invidious/views/channel/community.ecr b/src/invidious/views/channel/community.ecr index 1b1e8ab2d..28183f395 100644 --- a/src/invidious/views/channel/community.ecr +++ b/src/invidious/views/channel/community.ecr @@ -3,7 +3,7 @@ <% end %> -<% content_type = 2 %> +<% content_type = 3 %> <% sort_options = Tuple.new %> <%= rendered "components/channel-information" %> diff --git a/src/invidious/views/channel/featured_channels.ecr b/src/invidious/views/channel/featured_channels.ecr index 2aafa14f9..bddac5dfc 100644 --- a/src/invidious/views/channel/featured_channels.ecr +++ b/src/invidious/views/channel/featured_channels.ecr @@ -13,10 +13,9 @@
-

- <% if category.auxiliary_data.has_key?("view") %> - <% category_url_param = "?view=#{category.auxiliary_data["view"]}&shelf_id=#{category.auxiliary_data["shelf_id"]}" %> - + + <% if category.url %> + <%= category.title %> <%else%> diff --git a/src/invidious/views/channel/home.ecr b/src/invidious/views/channel/home.ecr index e69de29bb..ffe15ddf8 100644 --- a/src/invidious/views/channel/home.ecr +++ b/src/invidious/views/channel/home.ecr @@ -0,0 +1,60 @@ +<% content_for "header" do %> + <%= channel.author %> - Invidious + +<% end %> + +<% content_type = 0 %> +<% sort_options = Tuple.new %> +<%= rendered "components/channel-information" %> + +
+ <% items.each do | section | %> + <% # Channel trailer %> + <% if section.is_a? Video %> +
+ <% # Placeholder solution. A mini player should be placed here + %> +
+ +
+ + <% if section.length_seconds != 0 %> +

<%= recode_length_seconds(section.length_seconds) %>

+ <% end %> +
+ +
+ +
+ <%= HTML.escape(section.title) %> +

+ <%= translate(locale, "`x` views", number_to_short_text(section.views || 0)) %> + <%= translate(locale, "Shared `x` ago", recode_date(section.published, locale)) %> +

+
+ <%= section.description_html %> +
+ READ MORE +
+
+ <% else %> +
+
+ + <%= section.title %> + + +
+

<%= section.description_html %>

+
+ +
+ <% section.contents.each do |item| %> + <%= rendered "components/item" %> + <% end %> +
+
+
+ <% end %> + <% end %> +
\ No newline at end of file diff --git a/src/invidious/views/channel/playlists.ecr b/src/invidious/views/channel/playlists.ecr index 49dc749ef..4883666f8 100644 --- a/src/invidious/views/channel/playlists.ecr +++ b/src/invidious/views/channel/playlists.ecr @@ -3,7 +3,7 @@ <% end %> -<% content_type = 1 %> +<% content_type = 2 %> <%= rendered "components/channel-information" %>
diff --git a/src/invidious/views/components/channel-information.ecr b/src/invidious/views/components/channel-information.ecr index b6ec4bb37..9759a8e7d 100644 --- a/src/invidious/views/components/channel-information.ecr +++ b/src/invidious/views/components/channel-information.ecr @@ -60,23 +60,37 @@
+ <% if content_type == 0 %> +
  • + + <%= translate(locale, "Home") %> + +
    • + <% else %> +
  • + + <%= translate(locale, "Home") %> + +
    • + <% end %> + <% if !channel.auto_generated %> - <% if content_type == 0 %> + <% if content_type == 1 %>
  • - + <%= translate(locale, "Videos") %>
    • <% else %>
  • - + <%= translate(locale, "Videos") %>
    • <% end %> <% end %> - <% if content_type == 1 || channel.auto_generated %> + <% if content_type == 2 %>
  • <%= translate(locale, "Playlists") %> @@ -91,7 +105,7 @@ <% end %> <% if channel.tabs.has_key?("community") %> - <% if content_type == 2 %> + <% if content_type == 3 %>
  • <%= translate(locale, "Community") %> @@ -152,7 +166,7 @@
    • - <% if content_type == 0 || content_type == 1 %> + <% if content_type == 1 || content_type == 2 %> <% route = content_type == 1 ? "/playlists" : "" %> <% url = "/channel/#{channel.ucid + route}" %>