Merge 3c6019edd0 into 325561e755

This PR adds support for parsing the newer channel header format
(banner + subscription parsing)

Before this change:
* 0 subscribers
* No banner image

After this change:
* Example with Mr Breast channel: 299M
* Image banner is visible

Closes issue 4783

assets/js/pagination.js

@@ -0,0 +1,93 @@
+'use strict';
+
+const CURRENT_CONTINUATION = (new URL(document.location)).searchParams.get("continuation");
+const CONT_CACHE_KEY = `continuation_cache_${encodeURIComponent(window.location.pathname)}`;
+
+function get_data(){
+    return JSON.parse(sessionStorage.getItem(CONT_CACHE_KEY)) || [];
+}
+
+function save_data(){
+    const prev_data = get_data();
+    prev_data.push(CURRENT_CONTINUATION);
+
+    sessionStorage.setItem(CONT_CACHE_KEY, JSON.stringify(prev_data));
+}
+
+function button_press(){
+    let prev_data = get_data();
+    if (!prev_data.length) return null;
+
+    // Sanity check. Nowhere should the current continuation token exist in the cache 
+    // but it can happen when using the browser's back feature. As such we'd need to travel
+    // back to the point where the current continuation token first appears in order to 
+    // account for the rewind.
+    const conflict_at = prev_data.indexOf(CURRENT_CONTINUATION);
+    if (conflict_at != -1) {
+        prev_data.length = conflict_at;
+    }
+
+    const prev_ctoken = prev_data.pop();
+
+    // On the first page, the stored continuation token is null.
+    if (prev_ctoken === null) {
+        sessionStorage.removeItem(CONT_CACHE_KEY);
+        let url = set_continuation();
+        window.location.href = url;
+
+        return;
+    }
+
+    sessionStorage.setItem(CONT_CACHE_KEY, JSON.stringify(prev_data));
+    let url = set_continuation(prev_ctoken);
+
+    window.location.href = url;
+};
+
+// Method to set the current page's continuation token
+// Removes the continuation parameter when a continuation token is not given
+function set_continuation(prev_ctoken = null){
+    let url = window.location.href.split('?')[0];
+    let params = window.location.href.split('?')[1];
+    let url_params = new URLSearchParams(params);
+
+    if (prev_ctoken) {
+        url_params.set("continuation", prev_ctoken);
+    } else {
+        url_params.delete('continuation');
+    };
+
+    if(Array.from(url_params).length > 0){
+        return `${url}?${url_params.toString()}`;
+    } else {
+        return url;
+    }
+}
+
+addEventListener('DOMContentLoaded', function(){
+    const pagination_data = JSON.parse(document.getElementById('pagination-data').textContent);
+    const next_page_containers = document.getElementsByClassName("page-next-container");
+
+    for (let container of next_page_containers){
+        const next_page_button = container.getElementsByClassName("pure-button")
+
+        // exists?
+        if (next_page_button.length > 0){
+            next_page_button[0].addEventListener("click", save_data);
+        }
+    }
+
+    // Only add previous page buttons when not on the first page
+    if (CURRENT_CONTINUATION) {
+        const prev_page_containers = document.getElementsByClassName("page-prev-container")
+
+        for (let container of prev_page_containers) {
+            if (pagination_data.is_rtl) {
+                container.innerHTML = `<button class="pure-button pure-button-secondary">${pagination_data.prev_page}&nbsp;&nbsp;<i class="icon ion-ios-arrow-forward"></i></button>`
+            } else {
+                container.innerHTML = `<button class="pure-button pure-button-secondary"><i class="icon ion-ios-arrow-back"></i>&nbsp;&nbsp;${pagination_data.prev_page}</button>`
+            }
+            container.getElementsByClassName("pure-button")[0].addEventListener("click", button_press);
+        }
+    }
+});

assets/js/player.js

@@ -351,7 +351,12 @@ if (video_data.params.save_player_pos) {
     const rememberedTime = get_video_time();
     let lastUpdated = 0;
 
-    if(!hasTimeParam) set_seconds_after_start(rememberedTime);
+    if(!hasTimeParam) {
+      if (rememberedTime >= video_data.length_seconds - 20)
+        set_seconds_after_start(0);
+      else
+        set_seconds_after_start(rememberedTime);
+    }
 
     player.on('timeupdate', function () {
         const raw = player.currentTime();

locales/de.json

@@ -11,6 +11,7 @@
     "last": "neueste",
     "Next page": "Nächste Seite",
     "Previous page": "Vorherige Seite",
+    "First page": "Erste Seite",
     "Clear watch history?": "Verlauf löschen?",
     "New password": "Neues Passwort",
     "New passwords must match": "Neue Passwörter müssen übereinstimmen",

locales/en-US.json

@@ -33,6 +33,7 @@
     "last": "last",
     "Next page": "Next page",
     "Previous page": "Previous page",
+    "First page": "First page",
     "Clear watch history?": "Clear watch history?",
     "New password": "New password",
     "New passwords must match": "New passwords must match",

locales/ru.json

@@ -11,6 +11,7 @@
     "last": "последние",
     "Next page": "Следующая страница",
     "Previous page": "Предыдущая страница",
+    "First page": "Первая страница",
     "Clear watch history?": "Очистить историю просмотров?",
     "New password": "Новый пароль",
     "New passwords must match": "Новые пароли не совпадают",

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -67,7 +67,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos
@@ -151,7 +151,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Music")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

spec/invidious/videos/scheduled_live_extract_spec.cr

@@ -94,7 +94,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

src/invidious/channels/about.cr

@@ -72,6 +72,7 @@ def get_about_info(ucid, locale) : AboutChannel
 
     # Raises a KeyError on failure.
     banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+    banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
     banner = banners.try &.[-1]?.try &.["url"].as_s?
 
     # if banner.includes? "channels/c4/default_banner"
@@ -147,9 +148,17 @@ def get_about_info(ucid, locale) : AboutChannel
     end
   end
 
-  sub_count = initdata
-    .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s?
-    .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0
+  sub_count = 0
+
+  if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
+    metadata_rows.each do |row|
+      metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
+      if !metadata_part.nil?
+        sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
+      end
+      break if sub_count != 0
+    end
+  end
 
   AboutChannel.new(
     ucid: ucid,

src/invidious/frontend/pagination.cr

@@ -3,6 +3,24 @@ require "uri"
 module Invidious::Frontend::Pagination
   extend self
 
+  private def first_page(str : String::Builder, locale : String?, url : String)
+    str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">)
+
+    if locale_is_rtl?(locale)
+      # Inverted arrow ("first" points to the right)
+      str << translate(locale, "First page")
+      str << "&nbsp;&nbsp;"
+      str << %(<i class="icon ion-ios-arrow-forward"></i>)
+    else
+      # Regular arrow ("first" points to the left)
+      str << %(<i class="icon ion-ios-arrow-back"></i>)
+      str << "&nbsp;&nbsp;"
+      str << translate(locale, "First page")
+    end
+
+    str << "</a>"
+  end
+
   private def previous_page(str : String::Builder, locale : String?, url : String)
     # Link
     str << %(<a href=") << url << %(" class="pure-button pure-button-secondary">)
@@ -72,18 +90,24 @@ module Invidious::Frontend::Pagination
     end
   end
 
-  def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?)
+  def nav_ctoken(locale : String?, *, base_url : String | URI, ctoken : String?, first_page : Bool, params : URI::Params)
     return String.build do |str|
       str << %(<div class="h-box">\n)
       str << %(<div class="page-nav-container flexible">\n)
 
-      str << %(<div class="page-prev-container flex-left"></div>\n)
+      str << %(<div class="page-prev-container flex-left">)
+
+      if !first_page
+        self.first_page(str, locale, base_url.to_s)
+      end
+
+      str << %(</div>\n)
 
       str << %(<div class="page-next-container flex-right">)
 
       if !ctoken.nil?
-        params_next = URI::Params{"continuation" => ctoken}
-        url_next = HttpServer::Utils.add_params_to_url(base_url, params_next)
+        params["continuation"] = ctoken
+        url_next = HttpServer::Utils.add_params_to_url(base_url, params)
 
         self.next_page(str, locale, url_next.to_s)
       end

src/invidious/http_server/utils.cr

@@ -11,11 +11,12 @@ module Invidious::HttpServer
       params = url.query_params
       params["host"] = url.host.not_nil! # Should never be nil, in theory
       params["region"] = region if !region.nil?
+      url.query_params = params
 
       if absolute
-        return "#{HOST_URL}#{url.request_target}?#{params}"
+        return "#{HOST_URL}#{url.request_target}"
       else
-        return "#{url.request_target}?#{params}"
+        return url.request_target
       end
     end
 

src/invidious/jsonify/api_v1/video_json.cr

@@ -114,25 +114,31 @@ module Invidious::JSONify::APIv1
 
               json.field "projectionType", fmt["projectionType"]
 
-              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+              height = fmt["height"]?.try &.as_i
+              width = fmt["width"]?.try &.as_i
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
 
               # Livestream chunk infos
@@ -163,26 +169,31 @@ module Invidious::JSONify::APIv1
 
               json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
 
-              fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
-              if fmt_info
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+              height = fmt["height"]?.try &.as_i
+              width = fmt["width"]?.try &.as_i
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
             end
           end

src/invidious/playlists.cr

@@ -366,6 +366,8 @@ def fetch_playlist(plid : String)
 
     if text.includes? "video"
       video_count = text.gsub(/\D/, "").to_i? || 0
+    elsif text.includes? "episode"
+      video_count = text.gsub(/\D/, "").to_i? || 0
     elsif text.includes? "view"
       views = text.gsub(/\D/, "").to_i64? || 0_i64
     else

src/invidious/routes/api/v1/misc.cr

@@ -74,7 +74,9 @@ module Invidious::Routes::API::V1::Misc
       response = playlist.to_json(offset, video_id: video_id)
       json_response = JSON.parse(response)
 
-      if json_response["videos"].as_a[0]["index"] != offset
+      if json_response["videos"].as_a.empty?
+        json_response = JSON.parse(response)
+      elsif json_response["videos"].as_a[0]["index"] != offset
         offset = json_response["videos"].as_a[0]["index"].as_i
         lookback = offset < 50 ? offset : 50
         response = playlist.to_json(offset - lookback)

src/invidious/routes/api/v1/videos.cr

@@ -141,7 +141,11 @@ module Invidious::Routes::API::V1::Videos
           end
         end
       else
-        webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
+        uri = URI.parse(url)
+        query_params = uri.query_params
+        query_params["fmt"] = "vtt"
+        uri.query_params = query_params
+        webvtt = YT_POOL.client &.get(uri.request_target).body
 
         if webvtt.starts_with?("<?xml")
           webvtt = caption.timedtext_to_vtt(webvtt)

src/invidious/routes/preferences.cr

@@ -214,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
         statistics_enabled ||= "off"
         CONFIG.statistics_enabled = statistics_enabled == "on"
 
-        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
+        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
 
         File.write("config/config.yml", CONFIG.to_yaml)
       end

src/invidious/videos.cr

@@ -250,7 +250,7 @@ struct Video
   end
 
   def genre_url : String?
-    info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
+    info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
   end
 
   def is_vr : Bool?

src/invidious/videos/parser.cr

@@ -424,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
     # Video metadata
     "genre"     => JSON::Any.new(genre.try &.as_s || ""),
-    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
+    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
     "license"   => JSON::Any.new(license.try &.as_s || ""),
     # Music section
     "music" => JSON.parse(music_list.to_json),

src/invidious/views/channel.ecr

@@ -20,7 +20,9 @@
 
   page_nav_html = IV::Frontend::Pagination.nav_ctoken(locale,
     base_url: relative_url,
-    ctoken: next_continuation
+    ctoken: next_continuation,
+    first_page: continuation.nil?,
+    params: env.params.query,
   )
 %>
 
@@ -40,6 +42,8 @@
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
 <%- end -%>
 
+<script src="/js/pagination.js?v=<%= ASSET_COMMIT %>"></script>
+
 <link rel="alternate" href="<%= youtube_url %>">
 <title><%= author %> - Invidious</title>
 <% end %>

src/invidious/views/components/items_paginated.ecr

@@ -8,4 +8,14 @@
 
 <%= page_nav_html %>
 
+<script id="pagination-data" type="application/json">
+<%=
+{
+   "next_page" => translate(locale, "Next page"),
+   "prev_page" => translate(locale, "Previous page"),
+   "is_rtl"    => locale_is_rtl?(locale)
+}.to_pretty_json
+%>
+</script>
+
 <script src="/js/watched_indicator.js"></script>

src/invidious/views/user/preferences.ecr

@@ -310,7 +310,7 @@
 
                 <div class="pure-control-group">
                     <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label>
-                    <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>>
+                    <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>">
                 </div>
             <% end %>