Merge b5083a7746 into 4782a67038

Closes issue 4880

.github/workflows/build-stable-container.yml

@@ -47,9 +47,11 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: quay.io/invidious/invidious
+          flavor: |
+            latest=false
           tags: |
             type=semver,pattern={{version}}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=latest
           labels: |
             quay.expires-after=12w
 
@@ -71,10 +73,11 @@ jobs:
         with:
           images: quay.io/invidious/invidious
           flavor: |
+            latest=false
             suffix=-arm64
           tags: |
             type=semver,pattern={{version}}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+            type=raw,value=latest
           labels: |
             quay.expires-after=12w
 

CHANGELOG.md

@@ -1,5 +1,18 @@
 # CHANGELOG
 
+
+## v2.20240825.2 (2024-08-26)
+
+This releases fixes the container tags pushed on quay.io.
+Previously, the ARM64 build was released under the `latest` tag, instead of `latest-arm64`.
+
+### Full list of pull requests merged since the last release (newest first)
+
+CI: Fix docker container tags ([#4883], by @SamantazFox)
+
+[#4877]: https://github.com/iv-org/invidious/pull/4877
+
+
 ## v2.20240825.1 (2024-08-25)
 
 Add patch component to be [semver] compliant and make github actions happy.
@@ -8,8 +21,9 @@ Add patch component to be [semver] compliant and make github actions happy.
 
 ### Full list of pull requests merged since the last release (newest first)
 
-Allow manual trigger of release-container build (#4877, thanks @syeopite)
+Allow manual trigger of release-container build ([#4877], thanks @syeopite)
 
+[#4877]: https://github.com/iv-org/invidious/pull/4877
 
 
 ## v2.20240825.0 (2024-08-25)

src/invidious.cr

@@ -184,8 +184,9 @@ if CONFIG.popular_enabled
   Invidious::Jobs.register Invidious::Jobs::PullPopularVideosJob.new(PG_DB)
 end
 
-CONNECTION_CHANNEL = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
-Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
+NOTIFICATION_CHANNEL = ::Channel(VideoNotification).new(32)
+CONNECTION_CHANNEL   = ::Channel({Bool, ::Channel(PQ::Notification)}).new(32)
+Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(NOTIFICATION_CHANNEL, CONNECTION_CHANNEL, CONFIG.database_url)
 
 Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
 

src/invidious/channels/channels.cr

@@ -249,11 +249,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
 
     if was_insert
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
-      if CONFIG.enable_user_notifications
-        Invidious::Database::Users.add_notification(video)
-      else
-        Invidious::Database::Users.feed_needs_update(video)
-      end
+      NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
     else
       LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
     end
@@ -285,11 +281,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
         if Time.utc - video.published > 1.minute
           was_insert = Invidious::Database::ChannelVideos.insert(video)
           if was_insert
-            if CONFIG.enable_user_notifications
-              Invidious::Database::Users.add_notification(video)
-            else
-              Invidious::Database::Users.feed_needs_update(video)
-            end
+            NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
           end
         end
       end

src/invidious/database/users.cr

@@ -119,15 +119,15 @@ module Invidious::Database::Users
   #  Update (notifs)
   # -------------------
 
-  def add_notification(video : ChannelVideo)
+  def add_multiple_notifications(channel_id : String, video_ids : Array(String))
     request = <<-SQL
       UPDATE users
-      SET notifications = array_append(notifications, $1),
+      SET notifications = array_cat(notifications, $1),
           feed_needs_update = true
       WHERE $2 = ANY(subscriptions)
     SQL
 
-    PG_DB.exec(request, video.id, video.ucid)
+    PG_DB.exec(request, video_ids, channel_id)
   end
 
   def remove_notification(user : User, vid : String)
@@ -154,14 +154,14 @@ module Invidious::Database::Users
   #  Update (misc)
   # -------------------
 
-  def feed_needs_update(video : ChannelVideo)
+  def feed_needs_update(channel_id : String)
     request = <<-SQL
       UPDATE users
       SET feed_needs_update = true
       WHERE $1 = ANY(subscriptions)
     SQL
 
-    PG_DB.exec(request, video.ucid)
+    PG_DB.exec(request, channel_id)
   end
 
   def update_preferences(user : User)

src/invidious/jobs/notification_job.cr

@@ -1,8 +1,32 @@
+struct VideoNotification
+  getter video_id : String
+  getter channel_id : String
+  getter published : Time
+
+  def_hash @channel_id, @video_id
+
+  def ==(other)
+    video_id == other.video_id
+  end
+
+  def self.from_video(video : ChannelVideo) : self
+    VideoNotification.new(video.id, video.ucid, video.published)
+  end
+
+  def initialize(@video_id, @channel_id, @published)
+  end
+
+  def clone : VideoNotification
+    VideoNotification.new(video_id.clone, channel_id.clone, published.clone)
+  end
+end
+
 class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
+  private getter notification_channel : ::Channel(VideoNotification)
   private getter connection_channel : ::Channel({Bool, ::Channel(PQ::Notification)})
   private getter pg_url : URI
 
-  def initialize(@connection_channel, @pg_url)
+  def initialize(@notification_channel, @connection_channel, @pg_url)
   end
 
   def begin
@@ -10,6 +34,69 @@ class Invidious::Jobs::NotificationJob < Invidious::Jobs::BaseJob
 
     PG.connect_listen(pg_url, "notifications") { |event| connections.each(&.send(event)) }
 
+    # hash of channels to their videos (id+published) that need notifying
+    to_notify = Hash(String, Set(VideoNotification)).new(
+      ->(hash : Hash(String, Set(VideoNotification)), key : String) {
+        hash[key] = Set(VideoNotification).new
+      }
+    )
+    notify_mutex = Mutex.new()
+
+    # fiber to locally cache all incoming notifications (from pubsub webhooks and refresh channels job)
+    spawn do
+      begin
+        loop do
+          notification = notification_channel.receive
+          notify_mutex.lock
+          to_notify[notification.channel_id] << notification
+          notify_mutex.unlock
+        end
+      end
+    end
+    # fiber to regularly persist all cached notifications
+    spawn do
+      loop do
+        begin
+          LOGGER.debug("NotificationJob: waking up")
+          notify_mutex.lock
+          cloned = to_notify.clone
+          to_notify.clear
+          notify_mutex.unlock
+
+          cloned.each do |channel_id, notifications|
+            if notifications.empty?
+              next
+            end
+
+            LOGGER.info("NotificationJob: updating channel #{channel_id} with #{notifications.size} notifications")
+            if CONFIG.enable_user_notifications
+              video_ids = notifications.map { |n| n.video_id }
+              Invidious::Database::Users.add_multiple_notifications(channel_id, video_ids)
+              PG_DB.using_connection do |conn|
+                notifications.each do |n|
+                  # Deliver notifications to `/api/v1/auth/notifications`
+                  payload = {
+                    "topic"     => n.channel_id,
+                    "videoId"   => n.video_id,
+                    "published" => n.published.to_unix,
+                  }.to_json
+                  conn.exec("NOTIFY notifications, E'#{payload}'")
+                end
+              end
+            else
+              Invidious::Database::Users.feed_needs_update(channel_id)
+            end
+          end
+
+          LOGGER.trace("NotificationJob: Done, sleeping")
+        rescue ex
+          LOGGER.error("NotificationJob: #{ex.message}")
+        end
+        sleep 1.minute
+        Fiber.yield
+      end
+    end
+
     loop do
       action, connection = connection_channel.receive
 

src/invidious/routes/feeds.cr

@@ -425,16 +425,6 @@ module Invidious::Routes::Feeds
           next # skip this video since it raised an exception (e.g. it is a scheduled live event)
         end
 
-        if CONFIG.enable_user_notifications
-          # Deliver notifications to `/api/v1/auth/notifications`
-          payload = {
-            "topic"     => video.ucid,
-            "videoId"   => video.id,
-            "published" => published.to_unix,
-          }.to_json
-          PG_DB.exec("NOTIFY notifications, E'#{payload}'")
-        end
-
         video = ChannelVideo.new({
           id:                 id,
           title:              video.title,
@@ -450,11 +440,7 @@ module Invidious::Routes::Feeds
 
         was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)
         if was_insert
-          if CONFIG.enable_user_notifications
-            Invidious::Database::Users.add_notification(video)
-          else
-            Invidious::Database::Users.feed_needs_update(video)
-          end
+          NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
         end
       end
     end