Add support for the new channel layout - part 2 (#3419)

locales/en-US.json

@@ -404,9 +404,7 @@
     "`x` marked it with a ❤": "`x` marked it with a ❤",
     "Audio mode": "Audio mode",
     "Video mode": "Video mode",
-    "Videos": "Videos",
     "Playlists": "Playlists",
-    "Community": "Community",
     "search_filters_title": "Filters",
     "search_filters_date_label": "Upload date",
     "search_filters_date_option_none": "Any date",
@@ -472,5 +470,11 @@
     "crash_page_read_the_faq": "read the <a href=\"`x`\">Frequently Asked Questions (FAQ)</a>",
     "crash_page_search_issue": "searched for <a href=\"`x`\">existing issues on GitHub</a>",
     "crash_page_report_issue": "If none of the above helped, please <a href=\"`x`\">open a new issue on GitHub</a> (preferably in English) and include the following text in your message (do NOT translate that text):",
-    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>"
+    "error_video_not_in_playlist": "The requested video doesn't exist in this playlist. <a href=\"`x`\">Click here for the playlist home page.</a>",
+    "channel_tab_videos_label": "Videos",
+    "channel_tab_shorts_label": "Shorts",
+    "channel_tab_streams_label": "Livestreams",
+    "channel_tab_playlists_label": "Playlists",
+    "channel_tab_community_label": "Community",
+    "channel_tab_channels_label": "Channels"
 }

shard.lock

@@ -34,7 +34,7 @@ shards:
 
   protodec:
     git: https://github.com/iv-org/protodec.git
-    version: 0.1.4
+    version: 0.1.5
 
   radix:
     git: https://github.com/luislavena/radix.git

shard.yml

@@ -24,7 +24,7 @@ dependencies:
     version: ~> 0.6.1
   protodec:
     github: iv-org/protodec
-    version: ~> 0.1.4
+    version: ~> 0.1.5
   lsquic:
     github: iv-org/lsquic.cr
     version: ~> 2.18.1-2

spec/invidious/hashtag_spec.cr

@@ -4,7 +4,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 1)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page1")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)
@@ -57,7 +57,7 @@ Spectator.describe Invidious::Hashtag do
   it "parses richItemRenderer containers (test 2)" do
     # Enable mock
     test_content = load_mock("hashtag/martingarrix_page2")
-    videos = extract_items(test_content)
+    videos, _ = extract_items(test_content)
 
     expect(typeof(videos)).to eq(Array(SearchItem))
     expect(videos.size).to eq(60)

spec/invidious/helpers_spec.cr

@@ -23,12 +23,6 @@ Spectator.describe "Helper" do
     end
   end
 
-  describe "#produce_channel_playlists_url" do
-    it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
-      expect(produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW")).to eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
-    end
-  end
-
   describe "#produce_comment_continuation" do
     it "correctly produces a continuation token for comments" do
       expect(produce_comment_continuation("_cE8xSu6swE", "ADSJ_i2qvJeFtL0htmS5_K5Ctj3eGFVBMWL9Wd42o3kmUL6_mAzdLp85-liQZL0mYr_16BhaggUqX652Sv9JqV6VXinShSP-ZT6rL4NolPBaPXVtJsO5_rA_qE3GubAuLFw9uzIIXU2-HnpXbdgPLWTFavfX206hqWmmpHwUOrmxQV_OX6tYkM3ux3rPAKCDrT8eWL7MU3bLiNcnbgkW8o0h8KYLL_8BPa8LcHbTv8pAoNkjerlX1x7K4pqxaXPoyz89qNlnh6rRx6AXgAzzoHH1dmcyQ8CIBeOHg-m4i8ZxdX4dP88XWrIFg-jJGhpGP8JUMDgZgavxVx225hUEYZMyrLGler5em4FgbG62YWC51moLDLeYEA")).to eq("EkMSC19jRTh4U3U2c3dFyAEA4AEBogINKP___________wFAAMICHQgEGhdodHRwczovL3d3dy55b3V0dWJlLmNvbSIAGAYyjAMK9gJBRFNKX2kycXZKZUZ0TDBodG1TNV9LNUN0ajNlR0ZWQk1XTDlXZDQybzNrbVVMNl9tQXpkTHA4NS1saVFaTDBtWXJfMTZCaGFnZ1VxWDY1MlN2OUpxVjZWWGluU2hTUC1aVDZyTDROb2xQQmFQWFZ0SnNPNV9yQV9xRTNHdWJBdUxGdzl1eklJWFUyLUhucFhiZGdQTFdURmF2ZlgyMDZocVdtbXBId1VPcm14UVZfT1g2dFlrTTN1eDNyUEFLQ0RyVDhlV0w3TVUzYkxpTmNuYmdrVzhvMGg4S1lMTF84QlBhOExjSGJUdjhwQW9Oa2plcmxYMXg3SzRwcXhhWFBveXo4OXFObG5oNnJSeDZBWGdBenpvSEgxZG1jeVE4Q0lCZU9IZy1tNGk4WnhkWDRkUDg4WFdySUZnLWpKR2hwR1A4SlVNRGdaZ2F2eFZ4MjI1aFVFWVpNeXJMR2xlcjVlbTRGZ2JHNjJZV0M1MW1vTERMZVlFQSIPIgtfY0U4eFN1NnN3RTAAKBQ%3D")

src/invidious.cr

@@ -48,6 +48,13 @@ require "./invidious/search/*"
 require "./invidious/routes/**"
 require "./invidious/jobs/**"
 
+# Declare the base namespace for invidious
+module Invidious
+end
+
+# Simple alias to make code easier to read
+alias IV = Invidious
+
 CONFIG   = Config.load
 HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
 
@@ -172,7 +179,7 @@ if CONFIG.popular_enabled
   Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
 end
 
-CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
+CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
 Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
 
 Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new

src/invidious/channels/about.cr

@@ -16,12 +16,6 @@ record AboutChannel,
   tabs : Array(String),
   verified : Bool
 
-record AboutRelatedChannel,
-  ucid : String,
-  author : String,
-  author_url : String,
-  author_thumbnail : String
-
 def get_about_info(ucid, locale) : AboutChannel
   begin
     # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@@ -100,34 +94,46 @@ def get_about_info(ucid, locale) : AboutChannel
   total_views = 0_i64
   joined = Time.unix(0)
 
-  tabs = [] of String
+  tab_names = [] of String
 
-  tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?.try &.as_a?
+  if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
-  if !tabs_json.nil?
+    # Get the name of the tabs available on this channel
-    # Retrieve information from the tabs array. The index we are looking for varies between channels.
+    tab_names = tabs_json.as_a.compact_map do |entry|
-    tabs_json.each do |node|
+      name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
-      # Try to find the about section which is located in only one of the tabs.
-      channel_about_meta = node["tabRenderer"]?.try &.["content"]?.try &.["sectionListRenderer"]?
-        .try &.["contents"]?.try &.[0]?.try &.["itemSectionRenderer"]?.try &.["contents"]?
-          .try &.[0]?.try &.["channelAboutFullMetadataRenderer"]?
 
-      if !channel_about_meta.nil?
+      # This is a small fix to not add extra code on the HTML side
-        total_views = channel_about_meta["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+      # I.e, the URL for the "live" tab is .../streams, so use "streams"
-
+      # everywhere for the sake of simplicity
-        # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+      (name == "live") ? "streams" : name
-        joined = channel_about_meta["joinedDateText"]?.try &.["runs"]?.try &.as_a.reduce("") { |acc, nd| acc + nd["text"].as_s }
+    end
-          .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
-
+    # Get the currently active tab ("About")
-        # Normal Auto-generated channels
+    about_tab = extract_selected_tab(tabs_json)
-        # https://support.google.com/youtube/answer/2579942
+
-        # For auto-generated channels, channel_about_meta only has ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+    # Try to find the about metadata section
-        if (channel_about_meta["primaryLinks"]?.try &.size || 0) == 1 && (channel_about_meta["primaryLinks"][0]?) &&
+    channel_about_meta = about_tab.dig?(
-           (channel_about_meta["primaryLinks"][0]["title"]?.try &.["simpleText"]?.try &.as_s? || "") == "Auto-generated by YouTube"
+      "content",
-          auto_generated = true
+      "sectionListRenderer", "contents", 0,
-        end
+      "itemSectionRenderer", "contents", 0,
-      end
+      "channelAboutFullMetadataRenderer"
+    )
+
+    if !channel_about_meta.nil?
+      total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
+
+      # The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
+      joined = extract_text(channel_about_meta["joinedDateText"]?)
+        .try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
+
+      # Normal Auto-generated channels
+      # https://support.google.com/youtube/answer/2579942
+      # For auto-generated channels, channel_about_meta only has
+      # ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
+      auto_generated = (
+        (channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
+           extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube"
+      )
     end
-    tabs = tabs_json.reject { |node| node["tabRenderer"]?.nil? }.map(&.["tabRenderer"]["title"].as_s.downcase)
   end
 
   sub_count = initdata
@@ -148,46 +154,20 @@ def get_about_info(ucid, locale) : AboutChannel
     joined: joined,
     is_family_friendly: is_family_friendly,
     allowed_regions: allowed_regions,
-    tabs: tabs,
+    tabs: tab_names,
     verified: author_verified || false,
   )
 end
 
-def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel)
+def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
-  # params is {"2:string":"channels"} encoded
+  if continuation.nil?
-  channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
+    # params is {"2:string":"channels"} encoded
-
+    initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
-  tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any
+  else
-  tab = tabs.find(&.dig?("tabRenderer", "title").try(&.as_s?).try(&.== "Channels"))
+    initial_data = YoutubeAPI.browse(continuation)
-
-  return [] of AboutRelatedChannel if tab.nil?
-
-  items = tab.dig?(
-    "tabRenderer", "content",
-    "sectionListRenderer", "contents", 0,
-    "itemSectionRenderer", "contents", 0,
-    "gridRenderer", "items"
-  ).try &.as_a?
-
-  related = [] of AboutRelatedChannel
-  return related if (items.nil? || items.empty?)
-
-  items.each do |item|
-    renderer = item["gridChannelRenderer"]?
-    next if !renderer
-
-    related_id = renderer.dig("channelId").as_s
-    related_title = renderer.dig("title", "simpleText").as_s
-    related_author_url = renderer.dig("navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s
-    related_author_thumbnail = HelperExtractors.get_thumbnails(renderer)
-
-    related << AboutRelatedChannel.new(
-      ucid: related_id,
-      author: related_title,
-      author_url: related_author_url,
-      author_thumbnail: related_author_thumbnail,
-    )
   end
 
-  return related
+  items, continuation = extract_items(initial_data)
+
+  return items.select(SearchChannel), continuation
 end

src/invidious/channels/channels.cr

@@ -180,11 +180,16 @@ def fetch_channel(ucid, pull_all_videos : Bool)
 
   LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
 
-  page = 1
+  channel = InvidiousChannel.new({
+    id:         ucid,
+    author:     author,
+    updated:    Time.utc,
+    deleted:    false,
+    subscribed: nil,
+  })
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
-  initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+  videos, continuation = IV::Channel::Tabs.get_videos(channel)
-  videos = extract_videos(initial_data, author, ucid)
 
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//feed/entry").each do |entry|
@@ -197,7 +202,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
     views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
     views ||= 0_i64
 
-    channel_video = videos.select { |video| video.id == video_id }[0]?
+    channel_video = videos
+      .select(SearchVideo)
+      .select(&.id.== video_id)[0]?
 
     length_seconds = channel_video.try &.length_seconds
     length_seconds ||= 0
@@ -239,30 +246,25 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   end
 
   if pull_all_videos
-    page += 1
-
-    ids = [] of String
-
     loop do
-      initial_data = get_channel_videos_response(ucid, page, auto_generated: auto_generated)
+      # Keep fetching videos using the continuation token retrieved earlier
-      videos = extract_videos(initial_data, author, ucid)
+      videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
 
-      count = videos.size
+      count = 0
-      videos = videos.map { |video| ChannelVideo.new({
+      videos.select(SearchVideo).each do |video|
-        id:                 video.id,
+        count += 1
-        title:              video.title,
+        video = ChannelVideo.new({
-        published:          video.published,
+          id:                 video.id,
-        updated:            Time.utc,
+          title:              video.title,
-        ucid:               video.ucid,
+          published:          video.published,
-        author:             video.author,
+          updated:            Time.utc,
-        length_seconds:     video.length_seconds,
+          ucid:               video.ucid,
-        live_now:           video.live_now,
+          author:             video.author,
-        premiere_timestamp: video.premiere_timestamp,
+          length_seconds:     video.length_seconds,
-        views:              video.views,
+          live_now:           video.live_now,
-      }) }
+          premiere_timestamp: video.premiere_timestamp,
-
+          views:              video.views,
-      videos.each do |video|
+        })
-        ids << video.id
 
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
         # so since they don't provide a published date here we can safely ignore them.
@@ -279,17 +281,10 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       end
 
       break if count < 25
-      page += 1
+      sleep 500.milliseconds
     end
   end
 
-  channel = InvidiousChannel.new({
+  channel.updated = Time.utc
-    id:         ucid,
-    author:     author,
-    updated:    Time.utc,
-    deleted:    false,
-    subscribed: nil,
-  })
-
   return channel
 end

src/invidious/channels/playlists.cr

@@ -1,93 +1,28 @@
 def fetch_channel_playlists(ucid, author, continuation, sort_by)
   if continuation
-    response_json = YoutubeAPI.browse(continuation)
+    initial_data = YoutubeAPI.browse(continuation)
-    continuation_items = response_json["onResponseReceivedActions"]?
-      .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
-    return [] of SearchItem, nil if !continuation_items
-
-    items = [] of SearchItem
-    continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
-      extract_item(item, author, ucid).try { |t| items << t }
-    }
-
-    continuation = continuation_items.as_a.last["continuationItemRenderer"]?
-      .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
   else
-    url = "/channel/#{ucid}/playlists?flow=list&view=1"
+    params =
+      case sort_by
+      when "last", "last_added"
+        # Equivalent to "&sort=lad"
+        # {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYBCABMAE%3D"
+      when "oldest", "oldest_created"
+        # formerly "&sort=da"
+        # Not available anymore :c or maybe ??
+        # {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYAiABMAE%3D"
+        # {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
+        # "EglwbGF5bGlzdHMYASABMAE%3D"
+      when "newest", "newest_created"
+        # Formerly "&sort=dd"
+        # {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1}
+        "EglwbGF5bGlzdHMYAyABMAE%3D"
+      end
 
-    case sort_by
+    initial_data = YoutubeAPI.browse(ucid, params: params || "")
-    when "last", "last_added"
-      #
-    when "oldest", "oldest_created"
-      url += "&sort=da"
-    when "newest", "newest_created"
-      url += "&sort=dd"
-    else nil # Ignore
-    end
-
-    response = YT_POOL.client &.get(url)
-    initial_data = extract_initial_data(response.body)
-    return [] of SearchItem, nil if !initial_data
-
-    items = extract_items(initial_data, author, ucid)
-    continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?
   end
 
-  return items, continuation
+  return extract_items(initial_data, author, ucid)
-end
-
-# ## NOTE: DEPRECATED
-# Reason -> Unstable
-# The Protobuf object must be provided with an id of the last playlist from the current "page"
-# in order to fetch the next one accurately
-# (if the id isn't included, entries shift around erratically between pages,
-# leading to repetitions and skip overs)
-#
-# Since it's impossible to produce the appropriate Protobuf without an id being provided by the user,
-# it's better to stick to continuation tokens provided by the first request and onward
-def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated = false)
-  object = {
-    "80226972:embedded" => {
-      "2:string" => ucid,
-      "3:base64" => {
-        "2:string"  => "playlists",
-        "6:varint"  => 2_i64,
-        "7:varint"  => 1_i64,
-        "12:varint" => 1_i64,
-        "13:string" => "",
-        "23:varint" => 0_i64,
-      },
-    },
-  }
-
-  if cursor
-    cursor = Base64.urlsafe_encode(cursor, false) if !auto_generated
-    object["80226972:embedded"]["3:base64"].as(Hash)["15:string"] = cursor
-  end
-
-  if auto_generated
-    object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 0x32_i64
-  else
-    object["80226972:embedded"]["3:base64"].as(Hash)["4:varint"] = 1_i64
-    case sort
-    when "oldest", "oldest_created"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 2_i64
-    when "newest", "newest_created"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 3_i64
-    when "last", "last_added"
-      object["80226972:embedded"]["3:base64"].as(Hash)["3:varint"] = 4_i64
-    else nil # Ignore
-    end
-  end
-
-  object["80226972:embedded"]["3:string"] = Base64.urlsafe_encode(Protodec::Any.from_json(Protodec::Any.cast_json(object["80226972:embedded"]["3:base64"])))
-  object["80226972:embedded"].delete("3:base64")
-
-  continuation = object.try { |i| Protodec::Any.cast_json(i) }
-    .try { |i| Protodec::Any.from_json(i) }
-    .try { |i| Base64.urlsafe_encode(i) }
-    .try { |i| URI.encode_www_form(i) }
-
-  return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 end

src/invidious/channels/videos.cr

@@ -16,6 +16,14 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
     .try { |i| Base64.urlsafe_encode(i) }
     .try { |i| URI.encode_www_form(i) }
 
+  sort_by_numerical =
+    case sort_by
+    when "newest"  then 1_i64
+    when "popular" then 2_i64
+    when "oldest"  then 3_i64 # Broken as of 10/2022 :c
+    else                1_i64 # Fallback to "newest"
+    end
+
   object_inner_1 = {
     "110:embedded" => {
       "3:embedded" => {
@@ -24,7 +32,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
             "1:string" => object_inner_2_encoded,
             "2:string" => "00000000-0000-0000-0000-000000000000",
           },
-          "3:varint" => 1_i64,
+          "3:varint" => sort_by_numerical,
         },
       },
     },
@@ -52,34 +60,138 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
   return continuation
 end
 
-def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by = "newest")
-  continuation = produce_channel_videos_continuation(ucid, page,
-    auto_generated: auto_generated, sort_by: sort_by, v2: true)
-
-  return YoutubeAPI.browse(continuation)
-end
-
-def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
-  videos = [] of SearchVideo
-
-  # 2.times do |i|
-  # initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)
-  initial_data = get_channel_videos_response(ucid, 1, auto_generated: auto_generated, sort_by: sort_by)
-  videos = extract_videos(initial_data, author, ucid)
-  # end
-
-  return videos.size, videos
-end
-
-def get_latest_videos(ucid)
-  initial_data = get_channel_videos_response(ucid)
-  author = initial_data["metadata"]?.try &.["channelMetadataRenderer"]?.try &.["title"]?.try &.as_s
-
-  return extract_videos(initial_data, author, ucid)
-end
-
 # Used in bypass_captcha_job.cr
 def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
   continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
   return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 end
+
+module Invidious::Channel::Tabs
+  extend self
+
+  # -------------------
+  #  Regular videos
+  # -------------------
+
+  def make_initial_video_ctoken(ucid, sort_by) : String
+    return produce_channel_videos_continuation(ucid, sort_by: sort_by)
+  end
+
+  # Wrapper for AboutChannel, as we still need to call get_videos with
+  # an author name and ucid directly (e.g in RSS feeds).
+  # TODO: figure out how to get rid of that
+  def get_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    return get_videos(
+      channel.author, channel.ucid,
+      continuation: continuation, sort_by: sort_by
+    )
+  end
+
+  # Wrapper for InvidiousChannel, as we still need to call get_videos with
+  # an author name and ucid directly (e.g in RSS feeds).
+  # TODO: figure out how to get rid of that
+  def get_videos(channel : InvidiousChannel, *, continuation : String? = nil, sort_by = "newest")
+    return get_videos(
+      channel.author, channel.id,
+      continuation: continuation, sort_by: sort_by
+    )
+  end
+
+  def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
+    continuation ||= make_initial_video_ctoken(ucid, sort_by)
+    initial_data = YoutubeAPI.browse(continuation: continuation)
+
+    return extract_items(initial_data, author, ucid)
+  end
+
+  def get_60_videos(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
+    if continuation.nil?
+      # Fetch the first "page" of video
+      items, next_continuation = get_videos(channel, sort_by: sort_by)
+    else
+      # Fetch a "page" of videos using the given continuation token
+      items, next_continuation = get_videos(channel, continuation: continuation)
+    end
+
+    # If there is more to load, then load a second "page"
+    # and replace the previous continuation token
+    if !next_continuation.nil?
+      items_2, next_continuation = get_videos(channel, continuation: next_continuation)
+      items.concat items_2
+    end
+
+    return items, next_continuation
+  end
+
+  # -------------------
+  #  Shorts
+  # -------------------
+
+  private def fetch_shorts_data(ucid : String, continuation : String? = nil)
+    if continuation.nil?
+      # EgZzaG9ydHPyBgUKA5oBAA%3D%3D is the protobuf object to load "shorts"
+      # TODO: try to extract the continuation tokens that allows other sorting options
+      return YoutubeAPI.browse(ucid, params: "EgZzaG9ydHPyBgUKA5oBAA%3D%3D")
+    else
+      return YoutubeAPI.browse(continuation: continuation)
+    end
+  end
+
+  def get_shorts(channel : AboutChannel, continuation : String? = nil)
+    initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+
+    begin
+      # Try to parse the initial data fetched above
+      return extract_items(initial_data, channel.author, channel.ucid)
+    rescue ex : RetryOnceException
+      # Sometimes, for a completely unknown reason, the "reelItemRenderer"
+      # object is missing some critical information (it happens once in about
+      # 20 subsequent requests). Refreshing the page is required to properly
+      # show the "shorts" tab.
+      #
+      # In order to make the experience smoother for the user, we simulate
+      # said page refresh by fetching again the JSON. If that still doesn't
+      # work, we raise a BrokenTubeException, as something is really broken.
+      begin
+        initial_data = self.fetch_shorts_data(channel.ucid, continuation)
+        return extract_items(initial_data, channel.author, channel.ucid)
+      rescue ex : RetryOnceException
+        raise BrokenTubeException.new "reelPlayerHeaderSupportedRenderers"
+      end
+    end
+  end
+
+  # -------------------
+  #  Livestreams
+  # -------------------
+
+  def get_livestreams(channel : AboutChannel, continuation : String? = nil)
+    if continuation.nil?
+      # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
+      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
+    else
+      initial_data = YoutubeAPI.browse(continuation: continuation)
+    end
+
+    return extract_items(initial_data, channel.author, channel.ucid)
+  end
+
+  def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
+    if continuation.nil?
+      # Fetch the first "page" of streams
+      items, next_continuation = get_livestreams(channel)
+    else
+      # Fetch a "page" of streams using the given continuation token
+      items, next_continuation = get_livestreams(channel, continuation: continuation)
+    end
+
+    # If there is more to load, then load a second "page"
+    # and replace the previous continuation token
+    if !next_continuation.nil?
+      items_2, next_continuation = get_livestreams(channel, continuation: next_continuation)
+      items.concat items_2
+    end
+
+    return items, next_continuation
+  end
+end

src/invidious/exceptions.cr

@@ -33,3 +33,8 @@ end
 
 class VideoNotAvailableException < Exception
 end
+
+# Exception used to indicate that the JSON response from YT is missing
+# some important informations, and that the query should be sent again.
+class RetryOnceException < Exception
+end

src/invidious/frontend/channel_page.cr

@@ -0,0 +1,44 @@
+module Invidious::Frontend::ChannelPage
+  extend self
+
+  enum TabsAvailable
+    Videos
+    Shorts
+    Streams
+    Playlists
+    Community
+    Channels
+  end
+
+  def generate_tabs_links(locale : String, channel : AboutChannel, selected_tab : TabsAvailable)
+    return String.build(1500) do |str|
+      base_url = "/channel/#{channel.ucid}"
+
+      TabsAvailable.each do |tab|
+        # Ignore playlists, as it is not supported for auto-generated channels yet
+        next if (tab.playlists? && channel.auto_generated)
+
+        tab_name = tab.to_s.downcase
+
+        if channel.tabs.includes? tab_name
+          str << %(<div class="pure-u-1 pure-md-1-3">\n)
+
+          if tab == selected_tab
+            str << "\t<b>"
+            str << translate(locale, "channel_tab_#{tab_name}_label")
+            str << "</b>\n"
+          else
+            # Video tab doesn't have the last path component
+            url = tab.videos? ? base_url : "#{base_url}/#{tab_name}"
+
+            str << %(\t<a href=") << url << %(">)
+            str << translate(locale, "channel_tab_#{tab_name}_label")
+            str << "</a>\n"
+          end
+
+          str << "</div>"
+        end
+      end
+    end
+  end
+end

src/invidious/hashtag.cr

@@ -8,7 +8,8 @@ module Invidious::Hashtag
     client_config = YoutubeAPI::ClientConfig.new(region: region)
     response = YoutubeAPI.browse(continuation: ctoken, client_config: client_config)
 
-    return extract_items(response)
+    items, _ = extract_items(response)
+    return items
   end
 
   def generate_continuation(hashtag : String, cursor : Int)

src/invidious/helpers/serialized_yt_data.cr

@@ -265,4 +265,11 @@ class Category
   end
 end
 
+struct Continuation
+  getter token
+
+  def initialize(@token : String)
+  end
+end
+
 alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

src/invidious/jobs/notification_job.cr

@@ -1,12 +1,12 @@
 class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
-  private getter connection_channel : Channel({Bool, Channel(PQ::Notification)})
+  private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
   private getter pg_url : URI
 
   def initialize(@connection_channel, @pg_url)
   end
 
   def begin
-    connections = [] of Channel(PQ::Notification)
+    connections = [] of ::Channel(PQ::Notification)
 
     PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
 

src/invidious/jobs/refresh_channels_job.cr

@@ -8,7 +8,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
     max_fibers = CONFIG.channel_threads
     lim_fibers = max_fibers
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
     backoff = 2.minutes
 
     loop do

src/invidious/jobs/refresh_feeds_job.cr

@@ -7,7 +7,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
   def begin
     max_fibers = CONFIG.feed_threads
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
 
     loop do
       db.query("SELECT email FROM users WHERE feed_needs_update = true OR feed_needs_update IS NULL") do |rs|

src/invidious/jobs/subscribe_to_feeds_job.cr

@@ -12,7 +12,7 @@ class Invidious::Jobs::SubscribeToFeedsJob < Invidious::Jobs::BaseJob
     end
 
     active_fibers = 0
-    active_channel = Channel(Bool).new
+    active_channel = ::Channel(Bool).new
 
     loop do
       db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > interval '4 days' OR subscribed IS NULL") do |rs|

src/invidious/routes/api/v1/channels.cr

@@ -1,13 +1,7 @@
 module Invidious::Routes::API::V1::Channels
-  def self.home(env)
+  # Macro to avoid duplicating some code below
-    locale = env.get("preferences").as(Preferences).locale
+  # This sets the `channel` variable, or handles Exceptions.
-
+  private macro get_channel
-    env.response.content_type = "application/json"
-
-    ucid = env.params.url["ucid"]
-    sort_by = env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "newest"
-
     begin
       channel = get_about_info(ucid, locale)
     rescue ex : ChannelRedirect
@@ -18,17 +12,25 @@ module Invidious::Routes::API::V1::Channels
     rescue ex
       return error_json(500, ex)
     end
+  end
 
-    page = 1
+  def self.home(env)
-    if channel.auto_generated
+    locale = env.get("preferences").as(Preferences).locale
-      videos = [] of SearchVideo
+    ucid = env.params.url["ucid"]
-      count = 0
+
-    else
+    env.response.content_type = "application/json"
-      begin
+
-        count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+    # Use the private macro defined above.
-      rescue ex
+    channel = nil # Make the compiler happy
-        return error_json(500, ex)
+    get_channel()
-      end
+
+    # Retrieve "sort by" setting from URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+
+    begin
+      videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
+    rescue ex
+      return error_json(500, ex)
     end
 
     JSON.build do |json|
@@ -100,31 +102,13 @@ module Invidious::Routes::API::V1::Channels
           json.array do
             # Fetch related channels
             begin
-              related_channels = fetch_related_channels(channel)
+              related_channels, _ = fetch_related_channels(channel)
             rescue ex
-              related_channels = [] of AboutRelatedChannel
+              related_channels = [] of SearchChannel
             end
 
             related_channels.each do |related_channel|
-              json.object do
+              related_channel.to_json(locale, json)
-                json.field "author", related_channel.author
-                json.field "authorId", related_channel.ucid
-                json.field "authorUrl", related_channel.author_url
-
-                json.field "authorThumbnails" do
-                  json.array do
-                    qualities = {32, 48, 76, 100, 176, 512}
-
-                    qualities.each do |quality|
-                      json.object do
-                        json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
-                        json.field "width", quality
-                        json.field "height", quality
-                      end
-                    end
-                  end
-                end
-              end
             end
           end
         end # relatedChannels
@@ -134,61 +118,112 @@ module Invidious::Routes::API::V1::Channels
   end
 
   def self.latest(env)
-    locale = env.get("preferences").as(Preferences).locale
+    # Remove parameters that could affect this endpoint's behavior
+    env.params.query.delete("sort_by") if env.params.query.has_key?("sort_by")
+    env.params.query.delete("continuation") if env.params.query.has_key?("continuation")
 
-    env.response.content_type = "application/json"
+    return self.videos(env)
-
-    ucid = env.params.url["ucid"]
-
-    begin
-      videos = get_latest_videos(ucid)
-    rescue ex
-      return error_json(500, ex)
-    end
-
-    JSON.build do |json|
-      json.array do
-        videos.each do |video|
-          video.to_json(locale, json)
-        end
-      end
-    end
   end
 
   def self.videos(env)
     locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
 
     env.response.content_type = "application/json"
 
-    ucid = env.params.url["ucid"]
+    # Use the private macro defined above.
-    page = env.params.query["page"]?.try &.to_i?
+    channel = nil # Make the compiler happy
-    page ||= 1
+    get_channel()
-    sort_by = env.params.query["sort"]?.try &.downcase
+
-    sort_by ||= env.params.query["sort_by"]?.try &.downcase
+    # Retrieve some URL parameters
-    sort_by ||= "newest"
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
+    continuation = env.params.query["continuation"]?
 
     begin
-      channel = get_about_info(ucid, locale)
+      videos, next_continuation = Channel::Tabs.get_60_videos(
-    rescue ex : ChannelRedirect
+        channel, continuation: continuation, sort_by: sort_by
-      env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
+      )
-      return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
-    rescue ex : NotFoundException
-      return error_json(404, ex)
     rescue ex
       return error_json(500, ex)
     end
 
-    begin
+    return JSON.build do |json|
-      count, videos = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+      json.object do
-    rescue ex
+        json.field "videos" do
-      return error_json(500, ex)
+          json.array do
-    end
+            videos.each &.to_json(locale, json)
-
+          end
-    JSON.build do |json|
-      json.array do
-        videos.each do |video|
-          video.to_json(locale, json)
         end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
+  def self.shorts(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    # Retrieve continuation from URL parameters
+    continuation = env.params.query["continuation"]?
+
+    begin
+      videos, next_continuation = Channel::Tabs.get_shorts(
+        channel, continuation: continuation
+      )
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    return JSON.build do |json|
+      json.object do
+        json.field "videos" do
+          json.array do
+            videos.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
+  def self.streams(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the private macro defined above.
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    # Retrieve continuation from URL parameters
+    continuation = env.params.query["continuation"]?
+
+    begin
+      videos, next_continuation = Channel::Tabs.get_60_livestreams(
+        channel, continuation: continuation
+      )
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    return JSON.build do |json|
+      json.object do
+        json.field "videos" do
+          json.array do
+            videos.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
       end
     end
   end
@@ -204,16 +239,9 @@ module Invidious::Routes::API::V1::Channels
               env.params.query["sort_by"]?.try &.downcase ||
               "last"
 
-    begin
+    # Use the macro defined above
-      channel = get_about_info(ucid, locale)
+    channel = nil # Make the compiler happy
-    rescue ex : ChannelRedirect
+    get_channel()
-      env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
-      return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
-    rescue ex : NotFoundException
-      return error_json(404, ex)
-    rescue ex
-      return error_json(500, ex)
-    end
 
     items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
 
@@ -255,6 +283,37 @@ module Invidious::Routes::API::V1::Channels
     end
   end
 
+  def self.channels(env)
+    locale = env.get("preferences").as(Preferences).locale
+    ucid = env.params.url["ucid"]
+
+    env.response.content_type = "application/json"
+
+    # Use the macro defined above
+    channel = nil # Make the compiler happy
+    get_channel()
+
+    continuation = env.params.query["continuation"]?
+
+    begin
+      items, next_continuation = fetch_related_channels(channel, continuation)
+    rescue ex
+      return error_json(500, ex)
+    end
+
+    JSON.build do |json|
+      json.object do
+        json.field "relatedChannels" do
+          json.array do
+            items.each &.to_json(locale, json)
+          end
+        end
+
+        json.field "continuation", next_continuation if next_continuation
+      end
+    end
+  end
+
   def self.search(env)
     locale = env.get("preferences").as(Preferences).locale
     region = env.params.query["region"]?

src/invidious/routes/channels.cr

@@ -7,21 +7,19 @@ module Invidious::Routes::Channels
 
   def self.videos(env)
     data = self.fetch_basic_information(env)
-    if !data.is_a?(Tuple)
+    return data if !data.is_a?(Tuple)
-      return data
-    end
-    locale, user, subscriptions, continuation, ucid, channel = data
 
-    page = env.params.query["page"]?.try &.to_i?
+    locale, user, subscriptions, continuation, ucid, channel = data
-    page ||= 1
 
     sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if channel.auto_generated
       sort_options = {"last", "oldest", "newest"}
-      sort_by ||= "last"
 
-      items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+      items, next_continuation = fetch_channel_playlists(
+        channel.ucid, channel.author, continuation, (sort_by || "last")
+      )
+
       items.uniq! do |item|
         if item.responds_to?(:title)
           item.title
@@ -33,34 +31,85 @@ module Invidious::Routes::Channels
       items.each(&.author = "")
     else
       sort_options = {"newest", "oldest", "popular"}
-      sort_by ||= "newest"
 
-      count, items = get_60_videos(channel.ucid, channel.author, page, channel.auto_generated, sort_by)
+      # Fetch items and continuation token
+      items, next_continuation = Channel::Tabs.get_videos(
+        channel, continuation: continuation, sort_by: (sort_by || "newest")
+      )
     end
 
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Videos
+    templated "channel"
+  end
+
+  def self.shorts(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if !channel.tabs.includes? "shorts"
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    # TODO: support sort option for shorts
+    sort_by = ""
+    sort_options = [] of String
+
+    # Fetch items and continuation token
+    items, next_continuation = Channel::Tabs.get_shorts(
+      channel, continuation: continuation
+    )
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Shorts
+    templated "channel"
+  end
+
+  def self.streams(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if !channel.tabs.includes? "streams"
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    # TODO: support sort option for livestreams
+    sort_by = ""
+    sort_options = [] of String
+
+    # Fetch items and continuation token
+    items, next_continuation = Channel::Tabs.get_60_livestreams(
+      channel, continuation: continuation
+    )
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Streams
     templated "channel"
   end
 
   def self.playlists(env)
     data = self.fetch_basic_information(env)
-    if !data.is_a?(Tuple)
+    return data if !data.is_a?(Tuple)
-      return data
+
-    end
     locale, user, subscriptions, continuation, ucid, channel = data
 
     sort_options = {"last", "oldest", "newest"}
     sort_by = env.params.query["sort_by"]?.try &.downcase
-    sort_by ||= "last"
 
     if channel.auto_generated
       return env.redirect "/channel/#{channel.ucid}"
     end
 
-    items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
+    items, next_continuation = fetch_channel_playlists(
+      channel.ucid, channel.author, continuation, (sort_by || "last")
+    )
+
     items = items.select(SearchPlaylist).map(&.as(SearchPlaylist))
     items.each(&.author = "")
 
-    templated "playlists"
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Playlists
+    templated "channel"
   end
 
   def self.community(env)
@@ -74,12 +123,15 @@ module Invidious::Routes::Channels
     thin_mode = thin_mode == "true"
 
     continuation = env.params.query["continuation"]?
-    # sort_by = env.params.query["sort_by"]?.try &.downcase
 
     if !channel.tabs.includes? "community"
       return env.redirect "/channel/#{channel.ucid}"
     end
 
+    # TODO: support sort options for community posts
+    sort_by = ""
+    sort_options = [] of String
+
     begin
       items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
     rescue ex : InfoException
@@ -95,6 +147,26 @@ module Invidious::Routes::Channels
     templated "community"
   end
 
+  def self.channels(env)
+    data = self.fetch_basic_information(env)
+    return data if !data.is_a?(Tuple)
+
+    locale, user, subscriptions, continuation, ucid, channel = data
+
+    if channel.auto_generated
+      return env.redirect "/channel/#{channel.ucid}"
+    end
+
+    items, next_continuation = fetch_related_channels(channel, continuation)
+
+    # Featured/related channels can't be sorted
+    sort_options = [] of String
+    sort_by = nil
+
+    selected_tab = Frontend::ChannelPage::TabsAvailable::Channels
+    templated "channel"
+  end
+
   def self.about(env)
     data = self.fetch_basic_information(env)
     if !data.is_a?(Tuple)
@@ -125,7 +197,7 @@ module Invidious::Routes::Channels
     end
 
     selected_tab = env.request.path.split("/")[-1]
-    if ["home", "videos", "playlists", "community", "channels", "about"].includes? selected_tab
+    if {"home", "videos", "shorts", "streams", "playlists", "community", "channels", "about"}.includes? selected_tab
       url = "/channel/#{ucid}/#{selected_tab}"
     else
       url = "/channel/#{ucid}"

src/invidious/routing.cr

@@ -117,14 +117,17 @@ module Invidious::Routing
     get "/channel/:ucid", Routes::Channels, :home
     get "/channel/:ucid/home", Routes::Channels, :home
     get "/channel/:ucid/videos", Routes::Channels, :videos
+    get "/channel/:ucid/shorts", Routes::Channels, :shorts
+    get "/channel/:ucid/streams", Routes::Channels, :streams
     get "/channel/:ucid/playlists", Routes::Channels, :playlists
     get "/channel/:ucid/community", Routes::Channels, :community
+    get "/channel/:ucid/channels", Routes::Channels, :channels
     get "/channel/:ucid/about", Routes::Channels, :about
     get "/channel/:ucid/live", Routes::Channels, :live
     get "/user/:user/live", Routes::Channels, :live
     get "/c/:user/live", Routes::Channels, :live
 
-    ["", "/videos", "/playlists", "/community", "/about"].each do |path|
+    {"", "/videos", "/shorts", "/streams", "/playlists", "/community", "/about"}.each do |path|
       # /c/LinusTechTips
       get "/c/:user#{path}", Routes::Channels, :brand_redirect
       # /user/linustechtips | Not always the same as /c/
@@ -222,6 +225,10 @@ module Invidious::Routing
 
       # Channels
       get "/api/v1/channels/:ucid", {{namespace}}::Channels, :home
+      get "/api/v1/channels/:ucid/shorts", {{namespace}}::Channels, :shorts
+      get "/api/v1/channels/:ucid/streams", {{namespace}}::Channels, :streams
+      get "/api/v1/channels/:ucid/channels", {{namespace}}::Channels, :channels
+
       {% for route in {"videos", "latest", "playlists", "community", "search"} %}
         get "/api/v1/channels/#{{{route}}}/:ucid", {{namespace}}::Channels, :{{route}}
         get "/api/v1/channels/:ucid/#{{{route}}}", {{namespace}}::Channels, :{{route}}

src/invidious/search/processors.cr

@@ -9,7 +9,8 @@ module Invidious::Search
       client_config = YoutubeAPI::ClientConfig.new(region: query.region)
       initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
 
-      return extract_items(initial_data)
+      items, _ = extract_items(initial_data)
+      return items
     end
 
     # Search a youtube channel
@@ -30,16 +31,7 @@ module Invidious::Search
       continuation = produce_channel_search_continuation(ucid, query.text, query.page)
       response_json = YoutubeAPI.browse(continuation)
 
-      continuation_items = response_json["onResponseReceivedActions"]?
+      items, _ = extract_items(response_json, "", ucid)
-        .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
-
-      return [] of SearchItem if !continuation_items
-
-      items = [] of SearchItem
-      continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
-        extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
-      end
-
       return items
     end
 

src/invidious/views/channel.ecr

@@ -1,8 +1,24 @@
-<% ucid = channel.ucid %>
+<%-
-<% author = HTML.escape(channel.author) %>
+  ucid = channel.ucid
-<% channel_profile_pic = URI.parse(channel.author_thumbnail).request_target %>
+  author = HTML.escape(channel.author)
+  channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+  relative_url =
+    case selected_tab
+    when .shorts?    then "/channel/#{ucid}/shorts"
+    when .streams?   then "/channel/#{ucid}/streams"
+    when .playlists? then "/channel/#{ucid}/playlists"
+    when .channels?  then "/channel/#{ucid}/channels"
+    else
+      "/channel/#{ucid}"
+    end
+
+  youtube_url = "https://www.youtube.com#{relative_url}"
+  redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+-%>
 
 <% content_for "header" do %>
+<%- if selected_tab.videos? -%>
 <meta name="description" content="<%= channel.description %>">
 <meta property="og:site_name" content="Invidious">
 <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
@@ -14,91 +30,14 @@
 <meta name="twitter:title" content="<%= author %>">
 <meta name="twitter:description" content="<%= channel.description %>">
 <meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
-<link rel="alternate" href="https://www.youtube.com/channel/<%= ucid %>">
-<title><%= author %> - Invidious</title>
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
+<%- end -%>
+
+<link rel="alternate" href="<%= youtube_url %>">
+<title><%= author %> - Invidious</title>
 <% end %>
 
-<% if channel.banner %>
+<%= rendered "components/channel_info" %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= channel_profile_pic %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1-3">
-        <a href="https://www.youtube.com/channel/<%= ucid %>"><%= translate(locale, "View channel on YouTube") %></a>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.path %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-        <% if !channel.auto_generated %>
-            <div class="pure-u-1 pure-md-1-3">
-                <b><%= translate(locale, "Videos") %></b>
-            </div>
-        <% end %>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.auto_generated %>
-                <b><%= translate(locale, "Playlists") %></b>
-            <% else %>
-                <a href="/channel/<%= ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
-            <% end %>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
-            <% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3"></div>
-    <div class="pure-u-1-3">
-        <div class="pure-g" style="text-align:right">
-            <% sort_options.each do |sort| %>
-                <div class="pure-u-1 pure-md-1-3">
-                    <% if sort_by == sort %>
-                        <b><%= translate(locale, sort) %></b>
-                    <% else %>
-                        <a href="/channel/<%= ucid %>?page=<%= page %>&sort_by=<%= sort %>">
-                            <%= translate(locale, sort) %>
-                        </a>
-                    <% end %>
-                </div>
-            <% end %>
-        </div>
-    </div>
-</div>
 
 <div class="h-box">
     <hr>
@@ -111,17 +50,10 @@
 </div>
 
 <div class="pure-g h-box">
-    <div class="pure-u-1 pure-u-lg-1-5">
+    <div class="pure-u-1 pure-u-md-4-5"></div>
-        <% if page > 1 %>
-            <a href="/channel/<%= ucid %>?page=<%= page - 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
-                <%= translate(locale, "Previous page") %>
-            </a>
-        <% end %>
-    </div>
-    <div class="pure-u-1 pure-u-lg-3-5"></div>
     <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if count == 60 %>
+        <% if next_continuation %>
-            <a href="/channel/<%= ucid %>?page=<%= page + 1 %><% if sort_by != "newest" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
+            <a href="<%= relative_url %>?continuation=<%= next_continuation %><% if sort_options.any? sort_by %>&sort_by=<%= sort_by %><% end %>">
                 <%= translate(locale, "Next page") %>
             </a>
         <% end %>

src/invidious/views/community.ecr

@@ -1,71 +1,21 @@
-<% ucid = channel.ucid %>
+<%-
-<% author = HTML.escape(channel.author) %>
+  ucid = channel.ucid
+  author = HTML.escape(channel.author)
+  channel_profile_pic = URI.parse(channel.author_thumbnail).request_target
+
+  relative_url = "/channel/#{ucid}/community"
+  youtube_url = "https://www.youtube.com#{relative_url}"
+  redirect_url = Invidious::Frontend::Misc.redirect_url(env)
+
+  selected_tab = Invidious::Frontend::ChannelPage::TabsAvailable::Community
+-%>
 
 <% content_for "header" do %>
+<link rel="alternate" href="<%= youtube_url %>">
 <title><%= author %> - Invidious</title>
 <% end %>
 
-<% if channel.banner %>
+<%= rendered "components/channel_info" %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3" style="text-align:right">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= channel.ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1-3">
-        <a href="https://www.youtube.com/channel/<%= channel.ucid %>/community"><%= translate(locale, "View channel on YouTube") %></a>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-        <% if !channel.auto_generated %>
-            <div class="pure-u-1 pure-md-1-3">
-                <a href="/channel/<%= channel.ucid %>"><%= translate(locale, "Videos") %></a>
-            </div>
-        <% end %>
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="/channel/<%= channel.ucid %>/playlists"><%= translate(locale, "Playlists") %></a>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <b><%= translate(locale, "Community") %></b>
-            <% end %>
-        </div>
-    </div>
-    <div class="pure-u-2-3"></div>
-</div>
 
 <div class="h-box">
     <hr>

src/invidious/views/components/channel_info.ecr

@@ -0,0 +1,60 @@
+<% if channel.banner %>
+    <div class="h-box">
+        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
+    </div>
+
+    <div class="h-box">
+        <hr>
+    </div>
+<% end %>
+
+<div class="pure-g h-box">
+    <div class="pure-u-2-3">
+        <div class="channel-profile">
+            <img src="/ggpht<%= channel_profile_pic %>">
+            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+        </div>
+    </div>
+    <div class="pure-u-1-3">
+        <h3 style="text-align:right">
+            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
+        </h3>
+    </div>
+</div>
+
+<div class="h-box">
+    <div id="descriptionWrapper">
+        <p><span style="white-space:pre-wrap"><%= channel.description_html %></span></p>
+    </div>
+</div>
+
+<div class="h-box">
+    <% sub_count_text = number_to_short_text(channel.sub_count) %>
+    <%= rendered "components/subscribe_widget" %>
+</div>
+
+<div class="pure-g h-box">
+    <div class="pure-u-1-2">
+        <div class="pure-u-1 pure-md-1-3">
+            <a href="<%= youtube_url %>"><%= translate(locale, "View channel on YouTube") %></a>
+        </div>
+        <div class="pure-u-1 pure-md-1-3">
+            <a href="<%= redirect_url %>"><%= translate(locale, "Switch Invidious Instance") %></a>
+        </div>
+
+        <%= Invidious::Frontend::ChannelPage.generate_tabs_links(locale, channel, selected_tab) %>
+    </div>
+    <div class="pure-u-1-2">
+        <div class="pure-g" style="text-align:end">
+            <% sort_options.each do |sort| %>
+                <div class="pure-u-1 pure-md-1-3">
+                    <% if sort_by == sort %>
+                        <b><%= translate(locale, sort) %></b>
+                    <% else %>
+                        <a href="<%= relative_url %>?sort_by=<%= sort %>"><%= translate(locale, sort) %></a>
+                    <% end %>
+                </div>
+            <% end %>
+        </div>
+    </div>
+</div>

src/invidious/views/playlists.ecr

@@ -1,108 +0,0 @@
-<% ucid = channel.ucid %>
-<% author = HTML.escape(channel.author) %>
-
-<% content_for "header" do %>
-<title><%= author %> - Invidious</title>
-<% end %>
-
-<% if channel.banner %>
-    <div class="h-box">
-        <img style="width:100%" src="/ggpht<%= URI.parse(channel.banner.not_nil!.gsub("=w1060-", "=w1280-")).request_target %>">
-    </div>
-
-    <div class="h-box">
-        <hr>
-    </div>
-<% end %>
-
-<div class="pure-g h-box">
-    <div class="pure-u-2-3">
-        <div class="channel-profile">
-            <img src="/ggpht<%= URI.parse(channel.author_thumbnail).request_target %>">
-            <span><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3" style="text-align:right">
-        <h3 style="text-align:right">
-            <a href="/feed/channel/<%= ucid %>"><i class="icon ion-logo-rss"></i></a>
-        </h3>
-    </div>
-</div>
-
-<div class="h-box">
-    <div id="descriptionWrapper">
-        <p><span style="white-space:pre-wrap"><%= XML.parse_html(channel.description_html).xpath_node(%q(.//pre)).try &.content if !channel.description_html.empty? %></span></p>
-    </div>
-</div>
-
-<div class="h-box">
-    <% sub_count_text = number_to_short_text(channel.sub_count) %>
-    <%= rendered "components/subscribe_widget" %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-g pure-u-1-3">
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="https://www.youtube.com/channel/<%= ucid %>/playlists"><%= translate(locale, "View channel on YouTube") %></a>
-        </div>
-
-        <div class="pure-u-1 pure-md-1-3">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
-        </div>
-
-        <div class="pure-u-1 pure-md-1-3">
-            <a href="/channel/<%= ucid %>"><%= translate(locale, "Videos") %></a>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if !channel.auto_generated %>
-                <b><%= translate(locale, "Playlists") %></b>
-            <% end %>
-        </div>
-        <div class="pure-u-1 pure-md-1-3">
-            <% if channel.tabs.includes? "community" %>
-                <a href="/channel/<%= ucid %>/community"><%= translate(locale, "Community") %></a>
-            <% end %>
-        </div>
-    </div>
-    <div class="pure-u-1-3"></div>
-    <div class="pure-u-1-3">
-        <div class="pure-g" style="text-align:right">
-            <% {"last", "oldest", "newest"}.each do |sort| %>
-                <div class="pure-u-1 pure-md-1-3">
-                    <% if sort_by == sort %>
-                        <b><%= translate(locale, sort) %></b>
-                    <% else %>
-                        <a href="/channel/<%= ucid %>/playlists?sort_by=<%= sort %>">
-                            <%= translate(locale, sort) %>
-                        </a>
-                    <% end %>
-                </div>
-            <% end %>
-        </div>
-    </div>
-</div>
-
-<div class="h-box">
-    <hr>
-</div>
-
-<div class="pure-g">
-<% items.each do |item| %>
-    <%= rendered "components/item" %>
-<% end %>
-</div>
-
-<div class="pure-g h-box">
-    <div class="pure-u-1 pure-u-md-4-5"></div>
-    <div class="pure-u-1 pure-u-lg-1-5" style="text-align:right">
-        <% if continuation %>
-            <a href="/channel/<%= ucid %>/playlists?continuation=<%= continuation %><% if sort_by != "last" %>&sort_by=<%= URI.encode_www_form(sort_by) %><% end %>">
-                <%= translate(locale, "Next page") %>
-            </a>
-        <% end %>
-    </div>
-</div>

src/invidious/yt_backend/extractors.cr

@@ -7,7 +7,7 @@ require "../helpers/serialized_yt_data"
 private ITEM_CONTAINER_EXTRACTOR = {
   Extractors::YouTubeTabs,
   Extractors::SearchResults,
-  Extractors::Continuation,
+  Extractors::ContinuationContent,
 }
 
 private ITEM_PARSERS = {
@@ -18,8 +18,11 @@ private ITEM_PARSERS = {
   Parsers::CategoryRendererParser,
   Parsers::RichItemRendererParser,
   Parsers::ReelItemRendererParser,
+  Parsers::ContinuationItemRendererParser,
 }
 
+private alias InitialData = Hash(String, JSON::Any)
+
 record AuthorFallback, name : String, id : String
 
 # Namespace for logic relating to parsing InnerTube data into various datastructs.
@@ -345,14 +348,9 @@ private module Parsers
         content_container = item_contents["contents"]
       end
 
-      raw_contents = content_container["items"]?.try &.as_a
+      content_container["items"]?.try &.as_a.each do |item|
-      if !raw_contents.nil?
+        result = parse_item(item, author_fallback.name, author_fallback.id)
-        raw_contents.each do |item|
+        contents << result if result.is_a?(SearchItem)
-          result = extract_item(item)
-          if !result.nil?
-            contents << result
-          end
-        end
       end
 
       Category.new({
@@ -384,7 +382,9 @@ private module Parsers
     end
 
     private def self.parse(item_contents, author_fallback)
-      return VideoRendererParser.process(item_contents, author_fallback)
+      child = VideoRendererParser.process(item_contents, author_fallback)
+      child ||= ReelItemRendererParser.process(item_contents, author_fallback)
+      return child
     end
 
     def self.parser_name
@@ -408,9 +408,19 @@ private module Parsers
     private def self.parse(item_contents, author_fallback)
       video_id = item_contents["videoId"].as_s
 
-      video_details_container = item_contents.dig(
+      reel_player_overlay = item_contents.dig(
         "navigationEndpoint", "reelWatchEndpoint",
-        "overlay", "reelPlayerOverlayRenderer",
+        "overlay", "reelPlayerOverlayRenderer"
+      )
+
+      # Sometimes, the "reelPlayerOverlayRenderer" object is missing the
+      # important part of the response. We use this exception to tell
+      # the calling function to fetch the content again.
+      if !reel_player_overlay.as_h.has_key?("reelPlayerHeaderSupportedRenderers")
+        raise RetryOnceException.new
+      end
+
+      video_details_container = reel_player_overlay.dig(
         "reelPlayerHeaderSupportedRenderers",
         "reelPlayerHeaderRenderer"
       )
@@ -436,9 +446,9 @@ private module Parsers
 
       # View count
 
-      view_count_text = video_details_container.dig?("viewCountText", "simpleText")
+      # View count used to be in the reelWatchEndpoint, but that changed?
-      view_count_text ||= video_details_container
+      view_count_text = item_contents.dig?("viewCountText", "simpleText")
-        .dig?("viewCountText", "accessibility", "accessibilityData", "label")
+      view_count_text ||= video_details_container.dig?("viewCountText", "simpleText")
 
       view_count = view_count_text.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
 
@@ -450,8 +460,8 @@ private module Parsers
 
       regex_match = /- (?<min>\d+ minutes? )?(?<sec>\d+ seconds?)+ -/.match(a11y_data)
 
-      minutes = regex_match.try &.["min"].to_i(strict: false) || 0
+      minutes = regex_match.try &.["min"]?.try &.to_i(strict: false) || 0
-      seconds = regex_match.try &.["sec"].to_i(strict: false) || 0
+      seconds = regex_match.try &.["sec"]?.try &.to_i(strict: false) || 0
 
       duration = (minutes*60 + seconds)
 
@@ -475,6 +485,35 @@ private module Parsers
       return {{@type.name}}
     end
   end
+
+  # Parses an InnerTube continuationItemRenderer into a Continuation.
+  # Returns nil when the given object isn't a continuationItemRenderer.
+  #
+  # continuationItemRenderer contains various metadata ued to load more
+  # content (i.e when the user scrolls down). The interesting bit is the
+  # protobuf object known as the "continutation token". Previously, those
+  # were generated from sratch, but recent (as of 11/2022) Youtube changes
+  # are forcing us to extract them from replies.
+  #
+  module ContinuationItemRendererParser
+    def self.process(item : JSON::Any, author_fallback : AuthorFallback)
+      if item_contents = item["continuationItemRenderer"]?
+        return self.parse(item_contents)
+      end
+    end
+
+    private def self.parse(item_contents)
+      token = item_contents
+        .dig?("continuationEndpoint", "continuationCommand", "token")
+        .try &.as_s
+
+      return Continuation.new(token) if token
+    end
+
+    def self.parser_name
+      return {{@type.name}}
+    end
+  end
 end
 
 # The following are the extractors for extracting an array of items from
@@ -510,7 +549,7 @@ private module Extractors
   # }]
   #
   module YouTubeTabs
-    def self.process(initial_data : Hash(String, JSON::Any))
+    def self.process(initial_data : InitialData)
       if target = initial_data["twoColumnBrowseResultsRenderer"]?
         self.extract(target)
       end
@@ -575,7 +614,7 @@ private module Extractors
   # }
   #
   module SearchResults
-    def self.process(initial_data : Hash(String, JSON::Any))
+    def self.process(initial_data : InitialData)
       if target = initial_data["twoColumnSearchResultsRenderer"]?
         self.extract(target)
       end
@@ -608,8 +647,8 @@ private module Extractors
   # The way they are structured is too varied to be accurately written down here.
   # However, they all eventually lead to an array of parsable items after traversing
   # through the JSON structure.
-  module Continuation
+  module ContinuationContent
-    def self.process(initial_data : Hash(String, JSON::Any))
+    def self.process(initial_data : InitialData)
       if target = initial_data["continuationContents"]?
         self.extract(target)
       elsif target = initial_data["appendContinuationItemsAction"]?
@@ -691,8 +730,7 @@ end
 
 # Parses an item from Youtube's JSON response into a more usable structure.
 # The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
-def extract_item(item : JSON::Any, author_fallback : String? = "",
+def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
-                 author_id_fallback : String? = "")
   # We "allow" nil values but secretly use empty strings instead. This is to save us the
   # hassle of modifying every author_fallback and author_id_fallback arg usage
   # which is more often than not nil.
@@ -702,24 +740,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
   # Each parser automatically validates the data given to see if the data is
   # applicable to itself. If not nil is returned and the next parser is attempted.
   ITEM_PARSERS.each do |parser|
-    LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
+    LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
 
     if result = parser.process(item, author_fallback)
-      LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
+      LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
-
       return result
     else
-      LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
+      LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
     end
   end
 end
 
 # Parses multiple items from YouTube's initial JSON response into a more usable structure.
 # The end result is an array of SearchItem.
-def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
+#
-                  author_id_fallback : String? = nil) : Array(SearchItem)
+# This function yields the container so that items can be parsed separately.
-  items = [] of SearchItem
+#
-
+def extract_items(initial_data : InitialData, &block)
   if unpackaged_data = initial_data["contents"]?.try &.as_h
   elsif unpackaged_data = initial_data["response"]?.try &.as_h
   elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@@ -727,24 +764,37 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
     unpackaged_data = initial_data
   end
 
-  # This is identical to the parser cycling of extract_item().
+  # This is identical to the parser cycling of parse_item().
   ITEM_CONTAINER_EXTRACTOR.each do |extractor|
     LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
 
     if container = extractor.process(unpackaged_data)
       LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
       # Extract items in container
-      container.each do |item|
+      container.each { |item| yield item }
-        if parsed_result = extract_item(item, author_fallback, author_id_fallback)
-          items << parsed_result
-        end
-      end
-
-      break
     else
       LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
     end
   end
-
+end
-  return items
+
+# Wrapper using the block function above
+def extract_items(
+  initial_data : InitialData,
+  author_fallback : String? = nil,
+  author_id_fallback : String? = nil
+) : {Array(SearchItem), String?}
+  items = [] of SearchItem
+  continuation = nil
+
+  extract_items(initial_data) do |item|
+    parsed = parse_item(item, author_fallback, author_id_fallback)
+
+    case parsed
+    when .is_a?(Continuation) then continuation = parsed.token
+    when .is_a?(SearchItem)   then items << parsed
+    end
+  end
+
+  return items, continuation
 end

src/invidious/yt_backend/extractors_utils.cr

@@ -68,10 +68,10 @@ rescue ex
   return false
 end
 
-def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
+def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil) : Array(SearchVideo)
-  extracted = extract_items(initial_data, author_fallback, author_id_fallback)
+  extracted, _ = extract_items(initial_data, author_fallback, author_id_fallback)
 
-  target = [] of SearchItem
+  target = [] of (SearchItem | Continuation)
   extracted.each do |i|
     if i.is_a?(Category)
       i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
@@ -79,28 +79,11 @@ def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : Str
       target << i
     end
   end
-  return target.select(SearchVideo).map(&.as(SearchVideo))
+
+  return target.select(SearchVideo)
 end
 
 def extract_selected_tab(tabs)
   # Extract the selected tab from the array of tabs Youtube returns
   return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"]?.try &.as_bool)[0]["tabRenderer"]
 end
-
-def fetch_continuation_token(items : Array(JSON::Any))
-  # Fetches the continuation token from an array of items
-  return items.last["continuationItemRenderer"]?
-    .try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
-end
-
-def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
-  # Fetches the continuation token from initial data
-  if initial_data["onResponseReceivedActions"]?
-    continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
-  else
-    tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
-    continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
-  end
-
-  return fetch_continuation_token(continuation_items.as_a)
-end