WIP

This PR adds support for parsing the newer channel header format
(banner + subscription parsing)

Before this change:
* 0 subscribers
* No banner image

After this change:
* Example with Mr Breast channel: 299M
* Image banner is visible

Closes issue 4783

.ameba.yml

@@ -20,6 +20,9 @@ Lint/ShadowingOuterLocalVar:
   Excluded:
   - src/invidious/helpers/tokens.cr
 
+Lint/NotNil:
+  Enabled: false
+
 
 #
 # Style
@@ -31,6 +34,13 @@ Style/RedundantBegin:
 Style/RedundantReturn:
   Enabled: false
 
+Style/ParenthesesAroundCondition:
+  Enabled: false
+
+# This requires a rewrite of most data structs (and their usage) in Invidious.
+Style/QueryBoolMethods:
+  Enabled: false
+
 
 #
 # Metrics
@@ -39,50 +49,4 @@ Style/RedundantReturn:
 # Ignore function complexity (number of if/else & case/when branches)
 # For some functions that can hardly be simplified for now
 Metrics/CyclomaticComplexity:
-  Excluded:
+  Enabled: false
-  # get_about_info(ucid, locale) => [17/10]
-  - src/invidious/channels/about.cr
-
-  # fetch_channel_community(ucid, continuation, ...) => [34/10]
-  - src/invidious/channels/community.cr
-
-  # create_notification_stream(env, topics, connection_channel) => [14/10]
-  - src/invidious/helpers/helpers.cr:84:5
-
-  # get_index(plural_form, count) => [25/10]
-  - src/invidious/helpers/i18next.cr
-
-  # call(context) => [18/10]
-  - src/invidious/helpers/static_file_handler.cr
-
-  # show(env) => [38/10]
-  - src/invidious/routes/embed.cr
-
-  # get_video_playback(env) => [45/10]
-  - src/invidious/routes/video_playback.cr
-
-  # handle(env) => [40/10]
-  - src/invidious/routes/watch.cr
-
-  # playlist_ajax(env) => [24/10]
-  - src/invidious/routes/playlists.cr
-
-  # fetch_youtube_comments(id, cursor, ....) => [40/10]
-  # template_youtube_comments(comments, locale, ...) => [16/10]
-  # content_to_comment_html(content) => [14/10]
-  - src/invidious/comments.cr
-
-  # to_json(locale, json) => [21/10]
-  # extract_video_info(video_id, ...) => [44/10]
-  # process_video_params(query, preferences) => [20/10]
-  - src/invidious/videos.cr
-
-
-
-#src/invidious/playlists.cr:327:5
-#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [19/10]
-# fetch_playlist(plid : String)
-
-#src/invidious/playlists.cr:436:5
-#[C] Metrics/CyclomaticComplexity: Cyclomatic complexity too high [11/10]
-# extract_playlist_videos(initial_data : Hash(String, JSON::Any))

.github/workflows/build-nightly-container.yml

@@ -1,4 +1,4 @@
-name: Build and release container
+name: Build and release container directly from master
 
 on:
   push:
@@ -24,9 +24,9 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Crystal
-        uses: crystal-lang/install-crystal@v1.8.0
+        uses: crystal-lang/install-crystal@v1.8.2
         with:
-          crystal: 1.9.2
+          crystal: 1.12.2
 
       - name: Run lint
         run: |
@@ -58,7 +58,7 @@ jobs:
           images: quay.io/invidious/invidious
           tags: |
             type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
           labels: |
             quay.expires-after=12w
 
@@ -83,7 +83,7 @@ jobs:
             suffix=-arm64
           tags: |
             type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=master,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
           labels: |
             quay.expires-after=12w
 

.github/workflows/build-stable-container.yml

@@ -0,0 +1,90 @@
+name: Build and release container
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Crystal
+        uses: crystal-lang/install-crystal@v1.8.2
+        with:
+          crystal: 1.12.2
+
+      - name: Run lint
+        run: |
+          if ! crystal tool format --check; then
+            crystal tool format
+            git diff
+            exit 1
+          fi
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: arm64
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to registry
+        uses: docker/login-action@v3
+        with:
+          registry: quay.io
+          username: ${{ secrets.QUAY_USERNAME }}
+          password: ${{ secrets.QUAY_PASSWORD }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: quay.io/invidious/invidious
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+          labels: |
+            quay.expires-after=12w
+
+      - name: Build and push Docker AMD64 image for Push Event
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile
+          platforms: linux/amd64
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          build-args: |
+            "release=1"
+
+      - name: Docker meta
+        id: meta-arm64
+        uses: docker/metadata-action@v5
+        with:
+          images: quay.io/invidious/invidious
+          flavor: |
+            suffix=-arm64
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+          labels: |
+            quay.expires-after=12w
+
+      - name: Build and push Docker ARM64 image for Push Event
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: docker/Dockerfile.arm64
+          platforms: linux/arm64/v8
+          labels: ${{ steps.meta-arm64.outputs.labels }}
+          push: true
+          tags: ${{ steps.meta-arm64.outputs.tags }}
+          build-args: |
+            "release=1"

.github/workflows/ci.yml

@@ -38,10 +38,10 @@ jobs:
       matrix:
         stable: [true]
         crystal:
-          - 1.7.3
-          - 1.8.2
           - 1.9.2
           - 1.10.1
+          - 1.11.2
+          - 1.12.1
         include:
           - crystal: nightly
             stable: false
@@ -124,4 +124,28 @@ jobs:
       - name: Test Docker
         run: while curl -Isf http://localhost:3000; do sleep 1; done
 
+  ameba_lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
 
+      - name: Install Crystal
+        uses: crystal-lang/install-crystal@v1.8.0
+        with:
+          crystal: latest
+
+      - name: Cache Shards
+        uses: actions/cache@v3
+        with:
+          path: |
+            ./lib
+            ./bin
+          key: shards-${{ hashFiles('shard.lock') }}
+
+      - name: Install Shards
+        run: shards install
+
+      - name: Run Ameba linter
+        run: bin/ameba

assets/js/player.js

@@ -351,7 +351,12 @@ if (video_data.params.save_player_pos) {
     const rememberedTime = get_video_time();
     let lastUpdated = 0;
 
-    if(!hasTimeParam) set_seconds_after_start(rememberedTime);
+    if(!hasTimeParam) {
+      if (rememberedTime >= video_data.length_seconds - 20)
+        set_seconds_after_start(0);
+      else
+        set_seconds_after_start(rememberedTime);
+    }
 
     player.on('timeupdate', function () {
         const raw = player.currentTime();

config/config.example.yml

@@ -42,6 +42,24 @@ db:
 
 
 
+#########################################
+#
+#  Cache configuration
+#
+#########################################
+
+cache:
+  ##
+  ## URL of the caching server. To not use a caching server,
+  ## set to an empty string or leave empty.
+  ##
+  ## Note: The same "long" format as the 'db' parameter is
+  ## also supported.
+  ##
+  url: ""
+
+
+
 #########################################
 #
 #  Server config

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM crystallang/crystal:1.8.2-alpine AS builder
+FROM crystallang/crystal:1.12.1-alpine AS builder
 
 RUN apk add --no-cache sqlite-static yaml-static
 

docker/Dockerfile.arm64

@@ -1,5 +1,5 @@
-FROM alpine:3.18 AS builder
+FROM alpine:3.19 AS builder
-RUN apk add --no-cache 'crystal=1.8.2-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static
+RUN apk add --no-cache 'crystal=1.10.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-static zlib-static openssl-libs-static openssl-dev musl-dev xz-static
 
 ARG release
 

shard.lock

@@ -2,7 +2,7 @@ version: 2.0
 shards:
   ameba:
     git: https://github.com/crystal-ameba/ameba.git
-    version: 1.5.0
+    version: 1.6.1
 
   athena-negotiation:
     git: https://github.com/athena-framework/negotiation.git
@@ -32,6 +32,10 @@ shards:
     git: https://github.com/will/crystal-pg.git
     version: 0.24.0
 
+  pool:
+    git: https://github.com/ysbaddaden/pool.git
+    version: 0.2.4
+
   protodec:
     git: https://github.com/iv-org/protodec.git
     version: 0.1.5
@@ -40,6 +44,10 @@ shards:
     git: https://github.com/luislavena/radix.git
     version: 0.4.1
 
+  redis:
+    git: https://github.com/stefanwille/crystal-redis.git
+    version: 2.8.3
+
   spectator:
     git: https://github.com/icy-arctic-fox/spectator.git
     version: 0.10.4

shard.yml

@@ -10,32 +10,42 @@ targets:
     main: src/invidious.cr
 
 dependencies:
+  # Database
   pg:
     github: will/crystal-pg
     version: ~> 0.24.0
   sqlite3:
     github: crystal-lang/crystal-sqlite3
     version: ~> 0.18.0
+
+  # Web server
   kemal:
     github: kemalcr/kemal
     version: ~> 1.1.2
   kilt:
     github: jeromegn/kilt
     version: ~> 0.6.1
-  protodec:
-    github: iv-org/protodec
-    version: ~> 0.1.5
   athena-negotiation:
     github: athena-framework/negotiation
     version: ~> 0.1.1
 
+  # Youtube backend
+  protodec:
+    github: iv-org/protodec
+    version: ~> 0.1.5
+
+  # Caching
+  redis:
+    github: stefanwille/crystal-redis
+    version: ~> 2.8.3
+
 development_dependencies:
   spectator:
     github: icy-arctic-fox/spectator
     version: ~> 0.10.4
   ameba:
     github: crystal-ameba/ameba
-    version: ~> 1.5.0
+    version: ~> 1.6.1
 
 crystal: ">= 1.0.0, < 2.0.0"
 

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -67,7 +67,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos
@@ -151,7 +151,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Music")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

spec/invidious/videos/scheduled_live_extract_spec.cr

@@ -94,7 +94,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

src/invidious/cache.cr

@@ -0,0 +1,36 @@
+require "./cache/*"
+
+module Invidious::Cache
+  extend self
+
+  private INSTANCE = self.init(CONFIG.cache)
+
+  def init(cfg : Config::CacheConfig) : ItemStore
+    # Environment variable takes precedence over local config
+    url = ENV.fetch("INVIDIOUS_CACHE_URL", nil).try { |u| URI.parse(u) }
+    url ||= cfg.url
+    url ||= URI.new
+
+    # Determine cache type from URL scheme
+    type = StoreType.parse?(url.scheme || "none") || StoreType::None
+
+    case type
+    when .none?
+      return NullItemStore.new
+    when .redis?
+      if url.nil?
+        raise InvalidConfigException.new "Redis cache requires an URL."
+      end
+      return RedisItemStore.new(url)
+    else
+      raise InvalidConfigException.new "Invalid cache url. Only redis:// URL are currently supported."
+    end
+  end
+
+  # Shortcut methods to not have to specify INSTANCE everywhere in the code
+  {% for method in ["fetch", "store", "delete", "clear"] %}
+    def {{method.id}}(*args, **kwargs)
+      INSTANCE.{{method.id}}(*args, **kwargs)
+    end
+  {% end %}
+end

src/invidious/cache/item_store.cr

@@ -0,0 +1,22 @@
+require "./cacheable_item"
+
+module Invidious::Cache
+  # Abstract class from which any cached element should inherit
+  # Note: class is used here, instead of a module, in order to benefit
+  # from various compiler checks (e.g methods must be implemented)
+  abstract class ItemStore
+    # Retrieves an item from the store
+    # Returns nil if item wasn't found or is expired
+    abstract def fetch(key : String)
+
+    # Stores a given item into cache
+    abstract def store(key : String, value : String, expires : Time::Span)
+
+    # Prematurely deletes item(s) from the cache
+    abstract def delete(key : String)
+    abstract def delete(keys : Array(String))
+
+    # Removes all the items stored in the cache
+    abstract def clear
+  end
+end

src/invidious/cache/null_item_store.cr

@@ -0,0 +1,24 @@
+require "./item_store"
+
+module Invidious::Cache
+  class NullItemStore < ItemStore
+    def initialize
+    end
+
+    def fetch(key : String) : String?
+      return nil
+    end
+
+    def store(key : String, value : String, expires : Time::Span)
+    end
+
+    def delete(key : String)
+    end
+
+    def delete(keys : Array(String))
+    end
+
+    def clear
+    end
+  end
+end

src/invidious/cache/redis_item_store.cr

@@ -0,0 +1,33 @@
+require "./item_store"
+require "json"
+require "redis"
+
+module Invidious::Cache
+  class RedisItemStore < ItemStore
+    @redis : Redis::PooledClient
+
+    def initialize(url : URI)
+      @redis = Redis::PooledClient.new(url: url.to_s)
+    end
+
+    def fetch(key : String) : String?
+      return @redis.get(key)
+    end
+
+    def store(key : String, value : String, expires : Time::Span)
+      @redis.set(key, value, ex: expires.to_i)
+    end
+
+    def delete(key : String)
+      @redis.del(key)
+    end
+
+    def delete(keys : Array(String))
+      @redis.del(keys)
+    end
+
+    def clear
+      @redis.flushdb
+    end
+  end
+end

src/invidious/cache/store_type.cr

@@ -0,0 +1,6 @@
+module Invidious::Cache
+  enum StoreType
+    None
+    Redis
+  end
+end

src/invidious/channels/about.cr

@@ -72,6 +72,7 @@ def get_about_info(ucid, locale) : AboutChannel
 
     # Raises a KeyError on failure.
     banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+    banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
     banner = banners.try &.[-1]?.try &.["url"].as_s?
 
     # if banner.includes? "channels/c4/default_banner"
@@ -147,9 +148,17 @@ def get_about_info(ucid, locale) : AboutChannel
     end
   end
 
-  sub_count = initdata
+  sub_count = 0
-    .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s?
+
-    .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0
+  if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
+    metadata_rows.each do |row|
+      metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
+      if !metadata_part.nil?
+        sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
+      end
+      break if sub_count != 0
+    end
+  end
 
   AboutChannel.new(
     ucid: ucid,

src/invidious/channels/videos.cr

@@ -1,4 +1,4 @@
-def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
+def produce_channel_content_continuation(ucid, content_type, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
   object_inner_2 = {
     "2:0:embedded" => {
       "1:0:varint" => 0_i64,
@@ -16,6 +16,13 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
     .try { |i| Base64.urlsafe_encode(i) }
     .try { |i| URI.encode_www_form(i) }
 
+  content_type_numerical =
+    case content_type
+    when "videos"      then 15
+    when "livestreams" then 14
+    else                    15 # Fallback to "videos"
+    end
+
   sort_by_numerical =
     case sort_by
     when "newest"  then 1_i64
@@ -27,7 +34,7 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
   object_inner_1 = {
     "110:embedded" => {
       "3:embedded" => {
-        "15:embedded" => {
+        "#{content_type_numerical}:embedded" => {
           "1:embedded" => {
             "1:string" => object_inner_2_encoded,
           },
@@ -62,6 +69,10 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
   return continuation
 end
 
+def make_initial_content_ctoken(ucid, content_type, sort_by) : String
+  return produce_channel_content_continuation(ucid, content_type, sort_by: sort_by)
+end
+
 module Invidious::Channel::Tabs
   extend self
 
@@ -69,10 +80,6 @@ module Invidious::Channel::Tabs
   #  Regular videos
   # -------------------
 
-  def make_initial_video_ctoken(ucid, sort_by) : String
-    return produce_channel_videos_continuation(ucid, sort_by: sort_by)
-  end
-
   # Wrapper for AboutChannel, as we still need to call get_videos with
   # an author name and ucid directly (e.g in RSS feeds).
   # TODO: figure out how to get rid of that
@@ -94,7 +101,7 @@ module Invidious::Channel::Tabs
   end
 
   def get_videos(author : String, ucid : String, *, continuation : String? = nil, sort_by = "newest")
-    continuation ||= make_initial_video_ctoken(ucid, sort_by)
+    continuation ||= make_initial_content_ctoken(ucid, "videos", sort_by)
     initial_data = YoutubeAPI.browse(continuation: continuation)
 
     return extract_items(initial_data, author, ucid)
@@ -138,21 +145,18 @@ module Invidious::Channel::Tabs
   #  Livestreams
   # -------------------
 
-  def get_livestreams(channel : AboutChannel, continuation : String? = nil)
+  def get_livestreams(channel : AboutChannel, continuation : String? = nil, sort_by = "newest")
-    if continuation.nil?
+    continuation ||= make_initial_content_ctoken(channel.ucid, "livestreams", sort_by)
-      # EgdzdHJlYW1z8gYECgJ6AA%3D%3D is the protobuf object to load "streams"
+
-      initial_data = YoutubeAPI.browse(channel.ucid, params: "EgdzdHJlYW1z8gYECgJ6AA%3D%3D")
-    else
     initial_data = YoutubeAPI.browse(continuation: continuation)
-    end
 
     return extract_items(initial_data, channel.author, channel.ucid)
   end
 
-  def get_60_livestreams(channel : AboutChannel, continuation : String? = nil)
+  def get_60_livestreams(channel : AboutChannel, *, continuation : String? = nil, sort_by = "newest")
     if continuation.nil?
-      # Fetch the first "page" of streams
+      # Fetch the first "page" of stream
-      items, next_continuation = get_livestreams(channel)
+      items, next_continuation = get_livestreams(channel, sort_by: sort_by)
     else
       # Fetch a "page" of streams using the given continuation token
       items, next_continuation = get_livestreams(channel, continuation: continuation)

src/invidious/config.cr

@@ -1,12 +1,5 @@
-struct DBConfig
+require "yaml"
-  include YAML::Serializable
+require "./config/*"
-
-  property user : String
-  property password : String
-  property host : String
-  property port : Int32
-  property dbname : String
-end
 
 struct ConfigPreferences
   include YAML::Serializable
@@ -60,7 +53,7 @@ class Config
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
-  @[YAML::Field(converter: Preferences::TimeSpanConverter)]
+  @[YAML::Field(converter: IV::Config::TimeSpanConverter)]
   property channel_refresh_interval : Time::Span = 30.minutes
   # Number of threads to use for updating feeds
   property feed_threads : Int32 = 1
@@ -69,10 +62,10 @@ class Config
   # Default log level, valid YAML values are ints and strings, see src/invidious/helpers/logger.cr
   property log_level : LogLevel = LogLevel::Info
   # Database configuration with separate parameters (username, hostname, etc)
-  property db : DBConfig? = nil
+  property db : IV::Config::DBConfig? = nil
 
   # Database configuration using 12-Factor "Database URL" syntax
-  @[YAML::Field(converter: Preferences::URIConverter)]
+  @[YAML::Field(converter: IV::Config::URIConverter)]
   property database_url : URI = URI.parse("")
   # Use polling to keep decryption function up to date
   property decrypt_polling : Bool = false
@@ -81,6 +74,8 @@ class Config
 
   # Jobs config structure. See jobs.cr and jobs/base_job.cr
   property jobs = Invidious::Jobs::JobsConfig.new
+  # Cache configuration. See cache/cache.cr
+  property cache = Invidious::Config::CacheConfig.new
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
@@ -118,8 +113,9 @@ class Config
   property modified_source_code_url : String? = nil
 
   # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
-  @[YAML::Field(converter: Preferences::FamilyConverter)]
+  @[YAML::Field(converter: IV::Config::FamilyConverter)]
   property force_resolve : Socket::Family = Socket::Family::UNSPEC
+
   # Port to listen for connections (overridden by command line argument)
   property port : Int32 = 3000
   # Host to bind (overridden by command line argument)
@@ -131,7 +127,7 @@ class Config
   property use_innertube_for_captions : Bool = false
 
   # Saved cookies in "name1=value1; name2=value2..." format
-  @[YAML::Field(converter: Preferences::StringToCookies)]
+  @[YAML::Field(converter: IV::Config::CookiesConverter)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
 
   # Playlist length limit
@@ -214,14 +210,8 @@ class Config
     # Build database_url from db.* if it's not set directly
     if config.database_url.to_s.empty?
       if db = config.db
-        config.database_url = URI.new(
+        db.scheme = "postgres"
-          scheme: "postgres",
+        config.database_url = db.to_uri
-          user: db.user,
-          password: db.password,
-          host: db.host,
-          port: db.port,
-          path: db.dbname,
-        )
       else
         puts "Config: Either database_url or db.* is required"
         exit(1)

src/invidious/config/cache.cr

@@ -0,0 +1,14 @@
+require "../cache/store_type"
+
+module Invidious::Config
+  struct CacheConfig
+    include YAML::Serializable
+
+    @[YAML::Field(converter: IV::Config::URIConverter)]
+    property url : URI? = URI.new
+
+    # Required because of YAML serialization
+    def initialize
+    end
+  end
+end

src/invidious/config/converters.cr

@@ -0,0 +1,74 @@
+module Invidious::Config
+  module CookiesConverter
+    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+      unless node.is_a?(YAML::Nodes::Scalar)
+        node.raise "Expected scalar, not #{node.class}"
+      end
+
+      cookies = HTTP::Cookies.new
+      node.value.split(";").each do |cookie|
+        next if cookie.strip.empty?
+        name, value = cookie.split("=", 2)
+        cookies << HTTP::Cookie.new(name.strip, value.strip)
+      end
+
+      return cookies
+    end
+  end
+
+  module FamilyConverter
+    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
+      case value
+      when Socket::Family::UNSPEC then yaml.scalar nil
+      when Socket::Family::INET   then yaml.scalar "ipv4"
+      when Socket::Family::INET6  then yaml.scalar "ipv6"
+      when Socket::Family::UNIX   then raise "Invalid socket family #{value}"
+      end
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
+      if node.is_a?(YAML::Nodes::Scalar)
+        case node.value.downcase
+        when "ipv4" then Socket::Family::INET
+        when "ipv6" then Socket::Family::INET6
+        else
+          Socket::Family::UNSPEC
+        end
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+
+  module URIConverter
+    def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder)
+      yaml.scalar value.normalize!
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI
+      if node.is_a?(YAML::Nodes::Scalar)
+        URI.parse node.value
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+
+  module TimeSpanConverter
+    def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder)
+      return yaml.scalar value.total_minutes.to_i32
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span
+      if node.is_a?(YAML::Nodes::Scalar)
+        return decode_interval(node.value)
+      else
+        node.raise "Expected scalar, not #{node.class}"
+      end
+    end
+  end
+end

src/invidious/config/db.cr

@@ -0,0 +1,23 @@
+module Invidious::Config
+  struct DBConfig
+    include YAML::Serializable
+
+    property scheme : String
+    property user : String
+    property password : String
+    property host : String
+    property port : Int32
+    property dbname : String
+
+    def to_uri
+      return URI.new(
+        scheme: @scheme,
+        user: @user,
+        password: @password,
+        host: @host,
+        port: @port,
+        path: @dbname,
+      )
+    end
+  end
+end

src/invidious/database/base.cr

@@ -18,7 +18,6 @@ module Invidious::Database
     Invidious::Database.check_table("nonces", Nonce)
     Invidious::Database.check_table("session_ids", SessionId)
     Invidious::Database.check_table("users", User)
-    Invidious::Database.check_table("videos", Video)
 
     if cfg.cache_annotations
       Invidious::Database.check_table("annotations", Annotation)

src/invidious/exceptions.cr

@@ -38,3 +38,7 @@ end
 # some important informations, and that the query should be sent again.
 class RetryOnceException < Exception
 end
+
+# Exception used to indicate that the config file contains some errors
+class InvalidConfigException < Exception
+end

src/invidious/helpers/helpers.cr

@@ -74,7 +74,7 @@ def create_notification_stream(env, topics, connection_channel)
           published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
           video_id = TEST_IDS[rand(TEST_IDS.size)]
 
-          video = get_video(video_id)
+          video = Video.get(video_id)
           video.published = published
           response = JSON.parse(video.to_json(locale, nil))
 
@@ -133,7 +133,7 @@ def create_notification_stream(env, topics, connection_channel)
           next
         end
 
-        video = get_video(video_id)
+        video = Video.get(video_id)
         video.published = Time.unix(published)
         response = JSON.parse(video.to_json(locale, nil))
 

src/invidious/http_server/utils.cr

@@ -11,11 +11,12 @@ module Invidious::HttpServer
       params = url.query_params
       params["host"] = url.host.not_nil! # Should never be nil, in theory
       params["region"] = region if !region.nil?
+      url.query_params = params
 
       if absolute
-        return "#{HOST_URL}#{url.request_target}?#{params}"
+        return "#{HOST_URL}#{url.request_target}"
       else
-        return "#{url.request_target}?#{params}"
+        return url.request_target
       end
     end
 

src/invidious/jsonify/api_v1/video_json.cr

@@ -114,25 +114,31 @@ module Invidious::JSONify::APIv1
 
               json.field "projectionType", fmt["projectionType"]
 
-              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+              height = fmt["height"]?.try &.as_i
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+              width = fmt["width"]?.try &.as_i
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
 
               # Livestream chunk infos
@@ -163,26 +169,31 @@ module Invidious::JSONify::APIv1
 
               json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
 
-              fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+              height = fmt["height"]?.try &.as_i
-              if fmt_info
+              width = fmt["width"]?.try &.as_i
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
             end
           end

src/invidious/playlists.cr

@@ -366,6 +366,8 @@ def fetch_playlist(plid : String)
 
     if text.includes? "video"
       video_count = text.gsub(/\D/, "").to_i? || 0
+    elsif text.includes? "episode"
+      video_count = text.gsub(/\D/, "").to_i? || 0
     elsif text.includes? "view"
       views = text.gsub(/\D/, "").to_i64? || 0_i64
     else

src/invidious/routes/api/manifest.cr

@@ -6,14 +6,14 @@ module Invidious::Routes::API::Manifest
 
     local = env.params.query["local"]?.try &.== "true"
     id = env.params.url["id"]
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     # Since some implementations create playlists based on resolution regardless of different codecs,
     # we can opt to only add a source to a representation if it has a unique height within that representation
     unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }
 
     begin
-      video = get_video(id, region: region)
+      video = Video.get(id, region: region)
     rescue ex : NotFoundException
       haltf env, status_code: 404
     rescue ex

src/invidious/routes/api/v1/authenticated.cr

@@ -314,7 +314,7 @@ module Invidious::Routes::API::V1::Authenticated
     end
 
     begin
-      video = get_video(video_id)
+      video = Video.get(video_id)
     rescue ex : NotFoundException
       return error_json(404, ex)
     rescue ex

src/invidious/routes/api/v1/channels.cr

@@ -208,11 +208,12 @@ module Invidious::Routes::API::V1::Channels
     get_channel()
 
     # Retrieve continuation from URL parameters
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
     continuation = env.params.query["continuation"]?
 
     begin
       videos, next_continuation = Channel::Tabs.get_60_livestreams(
-        channel, continuation: continuation
+        channel, continuation: continuation, sort_by: sort_by
       )
     rescue ex
       return error_json(500, ex)
@@ -430,7 +431,7 @@ module Invidious::Routes::API::V1::Channels
 
   def self.search(env)
     locale = env.get("preferences").as(Preferences).locale
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     env.response.content_type = "application/json"
 

src/invidious/routes/api/v1/feeds.cr

@@ -4,7 +4,7 @@ module Invidious::Routes::API::V1::Feeds
 
     env.response.content_type = "application/json"
 
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
     trending_type = env.params.query["type"]?
 
     begin
@@ -31,7 +31,7 @@ module Invidious::Routes::API::V1::Feeds
 
     if !CONFIG.popular_enabled
       error_message = {"error" => "Administrator has disabled this endpoint."}.to_json
-      haltf env, 400, error_message
+      haltf env, 403, error_message
     end
 
     JSON.build do |json|

src/invidious/routes/api/v1/misc.cr

@@ -74,7 +74,9 @@ module Invidious::Routes::API::V1::Misc
       response = playlist.to_json(offset, video_id: video_id)
       json_response = JSON.parse(response)
 
-      if json_response["videos"].as_a[0]["index"] != offset
+      if json_response["videos"].as_a.empty?
+        json_response = JSON.parse(response)
+      elsif json_response["videos"].as_a[0]["index"] != offset
         offset = json_response["videos"].as_a[0]["index"].as_i
         lookback = offset < 50 ? offset : 50
         response = playlist.to_json(offset - lookback)

src/invidious/routes/api/v1/search.cr

@@ -1,7 +1,7 @@
 module Invidious::Routes::API::V1::Search
   def self.search(env)
     locale = env.get("preferences").as(Preferences).locale
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     env.response.content_type = "application/json"
 
@@ -24,7 +24,7 @@ module Invidious::Routes::API::V1::Search
 
   def self.search_suggestions(env)
     preferences = env.get("preferences").as(Preferences)
-    region = env.params.query["region"]? || preferences.region
+    region = find_region(env.params.query["region"]?) || preferences.region
 
     env.response.content_type = "application/json"
 
@@ -65,7 +65,7 @@ module Invidious::Routes::API::V1::Search
     page = env.params.query["page"]?.try &.to_i? || 1
 
     locale = env.get("preferences").as(Preferences).locale
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
     env.response.content_type = "application/json"
 
     begin

src/invidious/routes/api/v1/videos.cr

@@ -5,11 +5,11 @@ module Invidious::Routes::API::V1::Videos
     env.response.content_type = "application/json"
 
     id = env.params.url["id"]
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
     proxy = {"1", "true"}.any? &.== env.params.query["local"]?
 
     begin
-      video = get_video(id, region: region)
+      video = Video.get(id, region: region)
     rescue ex : NotFoundException
       return error_json(404, ex)
     rescue ex
@@ -25,7 +25,7 @@ module Invidious::Routes::API::V1::Videos
     env.response.content_type = "application/json"
 
     id = env.params.url["id"]
-    region = env.params.query["region"]? || env.params.body["region"]?
+    region = find_region(env.params.query["region"]? || env.params.body["region"]?)
 
     if id.nil? || id.size != 11 || !id.matches?(/^[\w-]+$/)
       return error_json(400, "Invalid video ID")
@@ -40,7 +40,7 @@ module Invidious::Routes::API::V1::Videos
     # getting video info.
 
     begin
-      video = get_video(id, region: region)
+      video = Video.get(id, region: region)
     rescue ex : NotFoundException
       haltf env, 404
     rescue ex
@@ -89,9 +89,14 @@ module Invidious::Routes::API::V1::Videos
 
     if CONFIG.use_innertube_for_captions
       params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
-      initial_data = YoutubeAPI.get_transcript(params)
 
-      webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
+      transcript = Invidious::Videos::Transcript.from_raw(
+        YoutubeAPI.get_transcript(params),
+        caption.language_code,
+        caption.auto_generated
+      )
+
+      webvtt = transcript.to_vtt
     else
       # Timedtext API handling
       url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
@@ -136,7 +141,11 @@ module Invidious::Routes::API::V1::Videos
           end
         end
       else
-        webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
+        uri = URI.parse(url)
+        query_params = uri.query_params
+        query_params["fmt"] = "vtt"
+        uri.query_params = query_params
+        webvtt = YT_POOL.client &.get(uri.request_target).body
 
         if webvtt.starts_with?("<?xml")
           webvtt = caption.timedtext_to_vtt(webvtt)
@@ -168,10 +177,10 @@ module Invidious::Routes::API::V1::Videos
     env.response.content_type = "application/json"
 
     id = env.params.url["id"]
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     begin
-      video = get_video(id, region: region)
+      video = Video.get(id, region: region)
     rescue ex : NotFoundException
       haltf env, 404
     rescue ex
@@ -297,7 +306,7 @@ module Invidious::Routes::API::V1::Videos
 
   def self.comments(env)
     locale = env.get("preferences").as(Preferences).locale
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     env.response.content_type = "application/json"
 

src/invidious/routes/before_all.cr

@@ -30,7 +30,7 @@ module Invidious::Routes::BeforeAll
 
     # Only allow the pages at /embed/* to be embedded
     if env.request.resource.starts_with?("/embed")
-      frame_ancestors = "'self' http: https:"
+      frame_ancestors = "'self' file: http: https:"
     else
       frame_ancestors = "'none'"
     end

src/invidious/routes/channels.cr

@@ -81,13 +81,12 @@ module Invidious::Routes::Channels
       return env.redirect "/channel/#{channel.ucid}"
     end
 
-    # TODO: support sort option for livestreams
+    sort_by = env.params.query["sort_by"]?.try &.downcase || "newest"
-    sort_by = ""
+    sort_options = {"newest", "oldest", "popular"}
-    sort_options = [] of String
 
     # Fetch items and continuation token
     items, next_continuation = Channel::Tabs.get_60_livestreams(
-      channel, continuation: continuation
+      channel, continuation: continuation, sort_by: sort_by
     )
 
     selected_tab = Frontend::ChannelPage::TabsAvailable::Streams

src/invidious/routes/embed.cr

@@ -130,7 +130,7 @@ module Invidious::Routes::Embed
     subscriptions ||= [] of String
 
     begin
-      video = get_video(id, region: params.region)
+      video = Video.get(id, region: params.region)
     rescue ex : NotFoundException
       return error_template(404, ex)
     rescue ex

src/invidious/routes/feeds.cr

@@ -48,7 +48,7 @@ module Invidious::Routes::Feeds
     trending_type = env.params.query["type"]?
     trending_type ||= "Default"
 
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
     region ||= env.get("preferences").as(Preferences).region
 
     begin
@@ -420,7 +420,7 @@ module Invidious::Routes::Feeds
         updated = Time.parse_rfc3339(entry.xpath_node("default:updated", namespaces).not_nil!.content)
 
         begin
-          video = get_video(id, force_refresh: true)
+          video = Video.get(id, force_refresh: true)
         rescue
           next # skip this video since it raised an exception (e.g. it is a scheduled live event)
         end

src/invidious/routes/playlists.cr

@@ -228,7 +228,7 @@ module Invidious::Routes::Playlists
     prefs = env.get("preferences").as(Preferences)
     locale = prefs.locale
 
-    region = env.params.query["region"]? || prefs.region
+    region = find_region(env.params.query["region"]?) || prefs.region
 
     user = env.get? "user"
     sid = env.get? "sid"
@@ -352,7 +352,7 @@ module Invidious::Routes::Playlists
       video_id = env.params.query["video_id"]
 
       begin
-        video = get_video(video_id)
+        video = Video.get(video_id)
       rescue ex : NotFoundException
         return error_json(404, ex)
       rescue ex

src/invidious/routes/preferences.cr

@@ -110,7 +110,7 @@ module Invidious::Routes::PreferencesRoute
     automatic_instance_redirect ||= "off"
     automatic_instance_redirect = automatic_instance_redirect == "on"
 
-    region = env.params.body["region"]?.try &.as(String)
+    region = find_region(env.params.body["region"]?)
 
     locale = env.params.body["locale"]?.try &.as(String)
     locale ||= CONFIG.default_user_preferences.locale
@@ -214,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
         statistics_enabled ||= "off"
         CONFIG.statistics_enabled = statistics_enabled == "on"
 
-        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
+        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
 
         File.write("config/config.yml", CONFIG.to_yaml)
       end

src/invidious/routes/search.cr

@@ -40,7 +40,7 @@ module Invidious::Routes::Search
     prefs = env.get("preferences").as(Preferences)
     locale = prefs.locale
 
-    region = env.params.query["region"]? || prefs.region
+    region = find_region(env.params.query["region"]?) || prefs.region
 
     query = Invidious::Search::Query.new(env.params.query, :regular, region)
 

src/invidious/routes/video_playback.cr

@@ -9,7 +9,7 @@ module Invidious::Routes::VideoPlayback
     mns ||= [] of String
 
     if query_params["region"]?
-      region = query_params["region"]
+      region = find_region(query_params["region"])
       query_params.delete("region")
     end
 
@@ -265,7 +265,7 @@ module Invidious::Routes::VideoPlayback
       return error_template(400, "Invalid itag")
     end
 
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
     local = (env.params.query["local"]? == "true")
 
     title = env.params.query["title"]?
@@ -275,7 +275,7 @@ module Invidious::Routes::VideoPlayback
     end
 
     begin
-      video = get_video(id, region: region)
+      video = Video.get(id, region: region)
     rescue ex : NotFoundException
       return error_template(404, ex)
     rescue ex

src/invidious/routes/watch.cr

@@ -3,7 +3,7 @@
 module Invidious::Routes::Watch
   def self.handle(env)
     locale = env.get("preferences").as(Preferences).locale
-    region = env.params.query["region"]?
+    region = find_region(env.params.query["region"]?)
 
     if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+")
       url = "/watch?" + env.params.query.to_s.gsub("%20", "").delete("+")
@@ -52,7 +52,7 @@ module Invidious::Routes::Watch
     env.params.query.delete_all("listen")
 
     begin
-      video = get_video(id, region: params.region)
+      video = Video.get(id, region: params.region)
     rescue ex : NotFoundException
       LOGGER.error("get_video not found: #{id} : #{ex.message}")
       return error_template(404, ex)

src/invidious/user/imports.cr

@@ -59,7 +59,7 @@ struct Invidious::User
           next if video_id == "Video Id"
 
           begin
-            video = get_video(video_id)
+            video = Video.get(video_id)
           rescue ex
             next
           end
@@ -133,7 +133,7 @@ struct Invidious::User
             next if !video_id
 
             begin
-              video = get_video(video_id, false)
+              video = Video.get(video_id)
             rescue ex
               next
             end

src/invidious/user/preferences.cr

@@ -1,6 +1,5 @@
 struct Preferences
   include JSON::Serializable
-  include YAML::Serializable
 
   property annotations : Bool = CONFIG.default_user_preferences.annotations
   property annotations_subscribed : Bool = CONFIG.default_user_preferences.annotations_subscribed
@@ -8,17 +7,14 @@ struct Preferences
   property automatic_instance_redirect : Bool = CONFIG.default_user_preferences.automatic_instance_redirect
 
   @[JSON::Field(converter: Preferences::StringToArray)]
-  @[YAML::Field(converter: Preferences::StringToArray)]
   property captions : Array(String) = CONFIG.default_user_preferences.captions
 
   @[JSON::Field(converter: Preferences::StringToArray)]
-  @[YAML::Field(converter: Preferences::StringToArray)]
   property comments : Array(String) = CONFIG.default_user_preferences.comments
   property continue : Bool = CONFIG.default_user_preferences.continue
   property continue_autoplay : Bool = CONFIG.default_user_preferences.continue_autoplay
 
   @[JSON::Field(converter: Preferences::BoolToString)]
-  @[YAML::Field(converter: Preferences::BoolToString)]
   property dark_mode : String = CONFIG.default_user_preferences.dark_mode
   property latest_only : Bool = CONFIG.default_user_preferences.latest_only
   property listen : Bool = CONFIG.default_user_preferences.listen
@@ -78,27 +74,6 @@ struct Preferences
         end
       end
     end
-
-    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      case node.value
-      when "true"
-        "dark"
-      when "false"
-        "light"
-      when ""
-        CONFIG.default_user_preferences.dark_mode
-      else
-        node.value
-      end
-    end
   end
 
   module ClampInt
@@ -109,58 +84,6 @@ struct Preferences
     def self.from_json(value : JSON::PullParser) : Int32
       value.read_int.clamp(0, MAX_ITEMS_PER_PAGE).to_i32
     end
-
-    def self.to_yaml(value : Int32, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Int32
-      node.value.clamp(0, MAX_ITEMS_PER_PAGE)
-    end
-  end
-
-  module FamilyConverter
-    def self.to_yaml(value : Socket::Family, yaml : YAML::Nodes::Builder)
-      case value
-      when Socket::Family::UNSPEC
-        yaml.scalar nil
-      when Socket::Family::INET
-        yaml.scalar "ipv4"
-      when Socket::Family::INET6
-        yaml.scalar "ipv6"
-      when Socket::Family::UNIX
-        raise "Invalid socket family #{value}"
-      end
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Socket::Family
-      if node.is_a?(YAML::Nodes::Scalar)
-        case node.value.downcase
-        when "ipv4"
-          Socket::Family::INET
-        when "ipv6"
-          Socket::Family::INET6
-        else
-          Socket::Family::UNSPEC
-        end
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
-  end
-
-  module URIConverter
-    def self.to_yaml(value : URI, yaml : YAML::Nodes::Builder)
-      yaml.scalar value.normalize!
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : URI
-      if node.is_a?(YAML::Nodes::Scalar)
-        URI.parse node.value
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
   end
 
   module ProcessString
@@ -171,14 +94,6 @@ struct Preferences
     def self.from_json(value : JSON::PullParser) : String
       HTML.escape(value.read_string[0, 100])
     end
-
-    def self.to_yaml(value : String, yaml : YAML::Nodes::Builder)
-      yaml.scalar value
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : String
-      HTML.escape(node.value[0, 100])
-    end
   end
 
   module StringToArray
@@ -202,73 +117,5 @@ struct Preferences
 
       result
     end
-
-    def self.to_yaml(value : Array(String), yaml : YAML::Nodes::Builder)
-      yaml.sequence do
-        value.each do |element|
-          yaml.scalar element
-        end
-      end
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(String)
-      begin
-        unless node.is_a?(YAML::Nodes::Sequence)
-          node.raise "Expected sequence, not #{node.class}"
-        end
-
-        result = [] of String
-        node.nodes.each do |item|
-          unless item.is_a?(YAML::Nodes::Scalar)
-            node.raise "Expected scalar, not #{item.class}"
-          end
-
-          result << HTML.escape(item.value[0, 100])
-        end
-      rescue ex
-        if node.is_a?(YAML::Nodes::Scalar)
-          result = [HTML.escape(node.value[0, 100]), ""]
-        else
-          result = ["", ""]
-        end
-      end
-
-      result
-    end
-  end
-
-  module StringToCookies
-    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
-      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
-      unless node.is_a?(YAML::Nodes::Scalar)
-        node.raise "Expected scalar, not #{node.class}"
-      end
-
-      cookies = HTTP::Cookies.new
-      node.value.split(";").each do |cookie|
-        next if cookie.strip.empty?
-        name, value = cookie.split("=", 2)
-        cookies << HTTP::Cookie.new(name.strip, value.strip)
-      end
-
-      cookies
-    end
-  end
-
-  module TimeSpanConverter
-    def self.to_yaml(value : Time::Span, yaml : YAML::Nodes::Builder)
-      return yaml.scalar value.total_minutes.to_i32
-    end
-
-    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time::Span
-      if node.is_a?(YAML::Nodes::Scalar)
-        return decode_interval(node.value)
-      else
-        node.raise "Expected scalar, not #{node.class}"
-      end
-    end
   end
 end

src/invidious/videos.cr

@@ -5,8 +5,6 @@ enum VideoType
 end
 
 struct Video
-  include DB::Serializable
-
   # Version of the JSON structure
   # It prevents us from loading an incompatible version from cache
   # (either newer or older, if instances with different versions run
@@ -16,23 +14,16 @@ struct Video
   # the `params` structure in videos/parser.cr!!!
   #
   SCHEMA_VERSION = 2
+  CACHE_KEY = "video_v#{SCHEMA_VERSION}"
 
   property id : String
-
-  @[DB::Field(converter: Video::JSONConverter)]
   property info : Hash(String, JSON::Any)
-  property updated : Time
 
-  @[DB::Field(ignore: true)]
   @captions = [] of Invidious::Videos::Captions::Metadata
 
-  @[DB::Field(ignore: true)]
   property adaptive_fmts : Array(Hash(String, JSON::Any))?
-
-  @[DB::Field(ignore: true)]
   property fmt_stream : Array(Hash(String, JSON::Any))?
 
-  @[DB::Field(ignore: true)]
   property description : String?
 
   module JSONConverter
@@ -41,6 +32,45 @@ struct Video
     end
   end
 
+  # Create new object from cache (JSON)
+  def initialize(@id, @info)
+  end
+
+  def self.get(id : String, *, force_refresh = false, region = nil)
+    key = "#{CACHE_KEY}:#{id}"
+    key += ":#{region}" if !region.nil?
+
+    # Fetch video from cache, unles a force refresh is requested
+    info = force_refresh ? nil : IV::Cache::INSTANCE.fetch(key)
+    updated = false
+
+    # Fetch video from youtube, if needed
+    if info.nil?
+      video = Video.new(id, fetch_video(id, region))
+      updated = true
+    else
+      video = Video.new(id, JSON.parse(info).as_h)
+
+      # If the video has premiered or the live has started, refresh the data.
+      if (video.live_now && video.published < Time.utc)
+        video = Video.new(id, fetch_video(id, region))
+        updated = true
+      end
+    end
+
+    # Store updated entry in cache
+    # TODO: finer cache control based on video type & publication date
+    if updated
+      if video.live_now || video.published < Time.utc
+        IV::Cache::INSTANCE.store(key, info.to_json, 10.minutes)
+      else
+        IV::Cache::INSTANCE.store(key, info.to_json, 2.hours)
+      end
+    end
+
+    return Video.new(id, info)
+  end
+
   # Methods for API v1 JSON
 
   def to_json(locale : String?, json : JSON::Builder)
@@ -250,7 +280,7 @@ struct Video
   end
 
   def genre_url : String?
-    info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
+    info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
   end
 
   def is_vr : Bool?
@@ -362,35 +392,6 @@ struct Video
   getset_bool isUpcoming
 end
 
-def get_video(id, refresh = true, region = nil, force_refresh = false)
-  if (video = Invidious::Database::Videos.select(id)) && !region
-    # If record was last updated over 10 minutes ago, or video has since premiered,
-    # refresh (expire param in response lasts for 6 hours)
-    if (refresh &&
-       (Time.utc - video.updated > 10.minutes) ||
-       (video.premiere_timestamp.try &.< Time.utc)) ||
-       force_refresh ||
-       video.schema_version != Video::SCHEMA_VERSION # cache control
-      begin
-        video = fetch_video(id, region)
-        Invidious::Database::Videos.update(video)
-      rescue ex
-        Invidious::Database::Videos.delete(id)
-        raise ex
-      end
-    end
-  else
-    video = fetch_video(id, region)
-    Invidious::Database::Videos.insert(video) if !region
-  end
-
-  return video
-rescue DB::Error
-  # Avoid common `DB::PoolRetryAttemptsExceeded` error and friends
-  # Note: All DB errors inherit from `DB::Error`
-  return fetch_video(id, region)
-end
-
 def fetch_video(id, region)
   info = extract_video_info(video_id: id)
 
@@ -408,13 +409,7 @@ def fetch_video(id, region)
     end
   end
 
-  video = Video.new({
+  return info
-    id:      id,
-    info:    info,
-    updated: Time.utc,
-  })
-
-  return video
 end
 
 def process_continuation(query, plid, id)

src/invidious/videos/parser.cr

@@ -424,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
     # Video metadata
     "genre"     => JSON::Any.new(genre.try &.as_s || ""),
-    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
+    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
     "license"   => JSON::Any.new(license.try &.as_s || ""),
     # Music section
     "music" => JSON.parse(music_list.to_json),

src/invidious/videos/regions.cr

@@ -25,3 +25,13 @@ REGIONS = {
   "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
   "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
 }
+
+# Utility function that searches in the array above for a given input.
+def find_region(reg : String?) : String?
+  return nil if reg.nil?
+
+  # Normalize input
+  region = (reg || "").upcase[0..1]
+
+  return REGIONS.find(&.== region)
+end

src/invidious/videos/transcript.cr

@@ -1,8 +1,26 @@
 module Invidious::Videos
-  # Namespace for methods primarily relating to Transcripts
+  # A `Transcripts` struct encapsulates a sequence of lines that together forms the whole transcript for a given YouTube video.
-  module Transcript
+  # These lines can be categorized into two types: section headings and regular lines representing content from the video.
-    record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+  struct Transcript
+    # Types
+    record HeadingLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+    record RegularLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+    alias TranscriptLine = HeadingLine | RegularLine
 
+    property lines : Array(TranscriptLine)
+
+    property language_code : String
+    property auto_generated : Bool
+
+    # User friendly label for the current transcript.
+    # Example: "English (auto-generated)"
+    property label : String
+
+    # Initializes a new Transcript struct with the contents and associated metadata describing it
+    def initialize(@lines : Array(TranscriptLine), @language_code : String, @auto_generated : Bool, @label : String)
+    end
+
+    # Generates a protobuf string to fetch the requested transcript from YouTube
     def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
       kind = auto_generated ? "asr" : ""
 
@@ -30,48 +48,79 @@ module Invidious::Videos
       return params
     end
 
-    def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
+    # Constructs a Transcripts struct from the initial YouTube response
-      # Convert into array of TranscriptLine
+    def self.from_raw(initial_data : Hash(String, JSON::Any), language_code : String, auto_generated : Bool)
-      lines = self.parse(initial_data)
+      transcript_panel = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+        "content", "transcriptSearchPanelRenderer")
 
+      segment_list = transcript_panel.dig("body", "transcriptSegmentListRenderer")
+
+      if !segment_list["initialSegments"]?
+        raise NotFoundException.new("Requested transcript does not exist")
+      end
+
+      # Extract user-friendly label for the current transcript
+
+      footer_language_menu = transcript_panel.dig?(
+        "footer", "transcriptFooterRenderer", "languageMenu", "sortFilterSubMenuRenderer", "subMenuItems"
+      )
+
+      if footer_language_menu
+        label = footer_language_menu.as_a.select(&.["selected"].as_bool)[0]["title"].as_s
+      else
+        label = language_code
+      end
+
+      # Extract transcript lines
+
+      initial_segments = segment_list["initialSegments"].as_a
+
+      lines = [] of TranscriptLine
+
+      initial_segments.each do |line|
+        if unpacked_line = line["transcriptSectionHeaderRenderer"]?
+          line_type = HeadingLine
+        else
+          unpacked_line = line["transcriptSegmentRenderer"]
+          line_type = RegularLine
+        end
+
+        start_ms = unpacked_line["startMs"].as_s.to_i.millisecond
+        end_ms = unpacked_line["endMs"].as_s.to_i.millisecond
+        text = extract_text(unpacked_line["snippet"]) || ""
+
+        lines << line_type.new(start_ms, end_ms, text)
+      end
+
+      return Transcript.new(
+        lines: lines,
+        language_code: language_code,
+        auto_generated: auto_generated,
+        label: label
+      )
+    end
+
+    # Converts transcript lines to a WebVTT file
+    #
+    # This is used within Invidious to replace subtitles
+    # as to workaround YouTube's rate-limited timedtext endpoint.
+    def to_vtt
       settings_field = {
         "Kind"     => "captions",
-        "Language" => target_language,
+        "Language" => @language_code,
       }
 
-      # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
       vtt = WebVTT.build(settings_field) do |vtt|
-        lines.each do |line|
+        @lines.each do |line|
+          # Section headers are excluded from the VTT conversion as to
+          # match the regular captions returned from YouTube as much as possible
+          next if line.is_a? HeadingLine
+
           vtt.cue(line.start_ms, line.end_ms, line.line)
         end
       end
 
       return vtt
     end
-
-    private def self.parse(initial_data : Hash(String, JSON::Any))
-      body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
-        "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
-        "initialSegments").as_a
-
-      lines = [] of TranscriptLine
-      body.each do |line|
-        # Transcript section headers. They are not apart of the captions and as such we can safely skip them.
-        if line.as_h.has_key?("transcriptSectionHeaderRenderer")
-          next
-        end
-
-        line = line["transcriptSegmentRenderer"]
-
-        start_ms = line["startMs"].as_s.to_i.millisecond
-        end_ms = line["endMs"].as_s.to_i.millisecond
-
-        text = extract_text(line["snippet"]) || ""
-
-        lines << TranscriptLine.new(start_ms, end_ms, text)
-      end
-
-      return lines
-    end
   end
 end

src/invidious/videos/video_preferences.cr

@@ -38,7 +38,9 @@ def process_video_params(query, preferences)
   preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
   quality = query["quality"]?
   quality_dash = query["quality_dash"]?
-  region = query["region"]?
+
+  region = find_region(query["region"]?)
+
   related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
   speed = query["speed"]?.try &.rchop("x").to_f?
   video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }

src/invidious/views/channel.ecr

@@ -30,13 +30,13 @@
 <meta property="og:site_name" content="Invidious">
 <meta property="og:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
 <meta property="og:title" content="<%= author %>">
-<meta property="og:image" content="/ggpht<%= channel_profile_pic %>">
+<meta property="og:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>">
 <meta property="og:description" content="<%= channel.description %>">
 <meta name="twitter:card" content="summary">
 <meta name="twitter:url" content="<%= HOST_URL %>/channel/<%= ucid %>">
 <meta name="twitter:title" content="<%= author %>">
 <meta name="twitter:description" content="<%= channel.description %>">
-<meta name="twitter:image" content="/ggpht<%= channel_profile_pic %>">
+<meta name="twitter:image" content="<%= HOST_URL %>/ggpht<%= channel_profile_pic %>">
 <link rel="alternate" type="application/rss+xml" title="RSS" href="/feed/channel/<%= ucid %>" />
 <%- end -%>
 

src/invidious/views/user/preferences.ecr

@@ -310,7 +310,7 @@
 
                 <div class="pure-control-group">
                     <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label>
-                    <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>>
+                    <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>">
                 </div>
             <% end %>
 

src/invidious/views/watch.ecr

@@ -10,7 +10,7 @@
 <meta property="og:site_name" content="<%= author %> | Invidious">
 <meta property="og:url" content="<%= HOST_URL %>/watch?v=<%= video.id %>">
 <meta property="og:title" content="<%= title %>">
-<meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg">
+<meta property="og:image" content="<%= HOST_URL %>/vi/<%= video.id %>/maxres.jpg">
 <meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
 <meta property="og:type" content="video.other">
 <meta property="og:video:url" content="<%= HOST_URL %>/embed/<%= video.id %>">

src/invidious/yt_backend/youtube_api.cr

@@ -5,9 +5,6 @@
 module YoutubeAPI
   extend self
 
-  private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
-  private ANDROID_API_KEY = "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w"
-
   # For Android versions, see https://en.wikipedia.org/wiki/Android_version_history
   private ANDROID_APP_VERSION = "19.14.42"
   private ANDROID_USER_AGENT  = "com.google.android.youtube/19.14.42 (Linux; U; Android 12; US) gzip"
@@ -52,7 +49,6 @@ module YoutubeAPI
       name:       "WEB",
       name_proto: "1",
       version:    "2.20240304.00.00",
-      api_key:    DEFAULT_API_KEY,
       screen:     "WATCH_FULL_SCREEN",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -62,7 +58,6 @@ module YoutubeAPI
       name:       "WEB_EMBEDDED_PLAYER",
       name_proto: "56",
       version:    "1.20240303.00.00",
-      api_key:    DEFAULT_API_KEY,
       screen:     "EMBED",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -72,7 +67,6 @@ module YoutubeAPI
       name:       "MWEB",
       name_proto: "2",
       version:    "2.20240304.08.00",
-      api_key:    DEFAULT_API_KEY,
       os_name:    "Android",
       os_version: ANDROID_VERSION,
       platform:   "MOBILE",
@@ -81,7 +75,6 @@ module YoutubeAPI
       name:       "WEB",
       name_proto: "1",
       version:    "2.20240304.00.00",
-      api_key:    DEFAULT_API_KEY,
       screen:     "EMBED",
       os_name:    "Windows",
       os_version: WINDOWS_VERSION,
@@ -94,7 +87,6 @@ module YoutubeAPI
       name:                "ANDROID",
       name_proto:          "3",
       version:             ANDROID_APP_VERSION,
-      api_key:             ANDROID_API_KEY,
       android_sdk_version: ANDROID_SDK_VERSION,
       user_agent:          ANDROID_USER_AGENT,
       os_name:             "Android",
@@ -105,13 +97,11 @@ module YoutubeAPI
       name:       "ANDROID_EMBEDDED_PLAYER",
       name_proto: "55",
       version:    ANDROID_APP_VERSION,
-      api_key:    "AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw",
     },
     ClientType::AndroidScreenEmbed => {
       name:                "ANDROID",
       name_proto:          "3",
       version:             ANDROID_APP_VERSION,
-      api_key:             DEFAULT_API_KEY,
       screen:              "EMBED",
       android_sdk_version: ANDROID_SDK_VERSION,
       user_agent:          ANDROID_USER_AGENT,
@@ -123,7 +113,6 @@ module YoutubeAPI
       name:                "ANDROID_TESTSUITE",
       name_proto:          "30",
       version:             ANDROID_TS_APP_VERSION,
-      api_key:             ANDROID_API_KEY,
       android_sdk_version: ANDROID_SDK_VERSION,
       user_agent:          ANDROID_TS_USER_AGENT,
       os_name:             "Android",
@@ -137,7 +126,6 @@ module YoutubeAPI
       name:         "IOS",
       name_proto:   "5",
       version:      IOS_APP_VERSION,
-      api_key:      "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc",
       user_agent:   IOS_USER_AGENT,
       device_make:  "Apple",
       device_model: "iPhone14,5",
@@ -149,7 +137,6 @@ module YoutubeAPI
       name:         "IOS_MESSAGES_EXTENSION",
       name_proto:   "66",
       version:      IOS_APP_VERSION,
-      api_key:      DEFAULT_API_KEY,
       user_agent:   IOS_USER_AGENT,
       device_make:  "Apple",
       device_model: "iPhone14,5",
@@ -161,7 +148,6 @@ module YoutubeAPI
       name:         "IOS_MUSIC",
       name_proto:   "26",
       version:      "6.42",
-      api_key:      "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s",
       user_agent:   "com.google.ios.youtubemusic/6.42 (iPhone14,5; U; CPU iOS 17_4 like Mac OS X;)",
       device_make:  "Apple",
       device_model: "iPhone14,5",
@@ -176,13 +162,11 @@ module YoutubeAPI
       name:       "TVHTML5",
       name_proto: "7",
       version:    "7.20240304.10.00",
-      api_key:    DEFAULT_API_KEY,
     },
     ClientType::TvHtml5ScreenEmbed => {
       name:       "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
       name_proto: "85",
       version:    "2.0",
-      api_key:    DEFAULT_API_KEY,
       screen:     "EMBED",
     },
   }
@@ -237,11 +221,6 @@ module YoutubeAPI
       HARDCODED_CLIENTS[@client_type][:version]
     end
 
-    # :ditto:
-    def api_key : String
-      HARDCODED_CLIENTS[@client_type][:api_key]
-    end
-
     # :ditto:
     def screen : String
       HARDCODED_CLIENTS[@client_type][:screen]? || ""
@@ -606,7 +585,7 @@ module YoutubeAPI
     client_config ||= DEFAULT_CLIENT_CONFIG
 
     # Query parameters
-    url = "#{endpoint}?key=#{client_config.api_key}&prettyPrint=false"
+    url = "#{endpoint}?prettyPrint=false"
 
     headers = HTTP::Headers{
       "Content-Type"              => "application/json; charset=UTF-8",