From 4d410d124ffd775db9f154e3d99a8058dc8aec30 Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Wed, 18 Jan 2023 00:33:55 +0100 Subject: [PATCH 1/7] Add prometheus metrics at /api/v1/metrics --- src/invidious/helpers/helpers.cr | 8 ++++++++ src/invidious/jobs/statistics_refresh_job.cr | 8 ++++++++ src/invidious/routes/api/v1/misc.cr | 5 +++++ src/invidious/routing.cr | 3 +++ 4 files changed, 24 insertions(+) diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6dc9860e..c103e484 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -225,3 +225,11 @@ def get_playback_statistic return tracker.as(Hash(String, Int64 | Float64)) end + +def to_prometheus_metrics(metrics_struct : Hash(String, Number)) : String + return String.build do |str| + metrics_struct.each.each do |key, value| + str << key << " " << value << "\n" + end + end +end diff --git a/src/invidious/jobs/statistics_refresh_job.cr b/src/invidious/jobs/statistics_refresh_job.cr index 72d1ce88..431fb829 100644 --- a/src/invidious/jobs/statistics_refresh_job.cr +++ b/src/invidious/jobs/statistics_refresh_job.cr @@ -27,6 +27,11 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob "playback" => {} of String => Int64 | Float64, } + STATISTICS_PROMETHEUS = { + "invidious_updated_at" => Time.utc.to_unix, + "invidious_last_channel_refreshed_at" => 0_i64, + } + private getter db : DB::Database def initialize(@db, @software_config : Hash(String, String)) @@ -59,6 +64,9 @@ class Invidious::Jobs::StatisticsRefreshJob < Invidious::Jobs::BaseJob users["activeHalfyear"] = Invidious::Database::Statistics.count_users_active_1m users["activeMonth"] = Invidious::Database::Statistics.count_users_active_6m + STATISTICS_PROMETHEUS["invidious_updated_at"] = Time.utc.to_unix + STATISTICS_PROMETHEUS["invidious_last_channel_refreshed_at"] = Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64 + STATISTICS["metadata"] = { "updatedAt" => Time.utc.to_unix, "lastChannelRefreshedAt" => Invidious::Database::Statistics.channel_last_update.try &.to_unix || 0_i64, diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index b42ecd1a..d4e65655 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -26,6 +26,11 @@ module Invidious::Routes::API::V1::Misc end end + def self.metrics(env) + env.response.content_type = "text/plain" + return to_prometheus_metrics(Invidious::Jobs::StatisticsRefreshJob::STATISTICS_PROMETHEUS) + end + # APIv1 currently uses the same logic for both # user playlists and Invidious playlists. This means that we can't # reasonably split them yet. This should be addressed in APIv2 diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index d6bd991c..27c3d199 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -310,6 +310,9 @@ module Invidious::Routing # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats + if CONFIG.statistics_enabled + get "/api/v1/metrics", {{namespace}}::Misc, :metrics + end get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes From 27dd94f60de8b48fa3a1afcb15ecb0c1d6445969 Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Sun, 19 Mar 2023 16:28:10 +0100 Subject: [PATCH 2/7] Collecting num of requests and handling time from each Kemal route --- src/invidious/metrics.cr | 48 +++++++++++++++++++++++++++++ src/invidious/routes/api/v1/misc.cr | 27 +++++++++++++++- src/invidious/routing.cr | 1 + 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 src/invidious/metrics.cr diff --git a/src/invidious/metrics.cr b/src/invidious/metrics.cr new file mode 100644 index 00000000..8007ca7d --- /dev/null +++ b/src/invidious/metrics.cr @@ -0,0 +1,48 @@ +# Module containing an optional Kemal handler, which can be used +# to collect metrics about the usage of various Invidious routes. +module Metrics + record MetricLabels, request_method : String, request_route : String, response_code : Int32 + + # Counts how many a given route was used + REQUEST_COUNTERS = Hash(MetricLabels, Int64).new + + # Counts how much time was used to handle requests to each route + REQUEST_DURATION_SECONDS_SUMS = Hash(MetricLabels, Float32).new + + # The handler which will record metrics when registered in a Kemal application + METRICS_COLLECTOR = RouteMetricsCollector.new(REQUEST_COUNTERS, REQUEST_DURATION_SECONDS_SUMS) + + class RouteMetricsCollector < Kemal::Handler + def initialize( + @num_of_request_counters : Hash(MetricLabels, Int64), + @request_duration_seconds_sums : Hash(MetricLabels, Float32) + ) + end + + def call(context : HTTP::Server::Context) + request_handling_started = Time.utc + begin + call_next(context) + ensure + request_handling_finished = Time.utc + request_path = context.route.path + request_method = context.request.method + seconds_spent_handling = (request_handling_finished - request_handling_started).to_f + response_status = context.response.status_code.to_i + + LOGGER.trace("Collecting metrics: handling #{request_method} #{request_path} took #{seconds_spent_handling}s and finished with status #{response_status}") + metric_key = MetricLabels.new request_path, request_method, response_status + + unless @num_of_request_counters.has_key?(metric_key) + @num_of_request_counters[metric_key] = 0 + end + @num_of_request_counters[metric_key] += 1 + + unless @request_duration_seconds_sums.has_key?(metric_key) + @request_duration_seconds_sums[metric_key] = 0.0 + end + @request_duration_seconds_sums[metric_key] += seconds_spent_handling + end + end + end +end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index d4e65655..d827b622 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -1,3 +1,5 @@ +require "../../../metrics.cr" + module Invidious::Routes::API::V1::Misc # Stats API endpoint for Invidious def self.stats(env) @@ -28,7 +30,30 @@ module Invidious::Routes::API::V1::Misc def self.metrics(env) env.response.content_type = "text/plain" - return to_prometheus_metrics(Invidious::Jobs::StatisticsRefreshJob::STATISTICS_PROMETHEUS) + + return String.build do |str| + Metrics::REQUEST_COUNTERS.each do |metric_labels, value| + str << "http_requests_total{" + str << "method=\"" << metric_labels.request_method << "\" " + str << "route=\"" << metric_labels.request_route << "\" " + str << "response_code=\"" << metric_labels.response_code << "\"" + str << "} " + str << value << "\n" + end + + Metrics::REQUEST_DURATION_SECONDS_SUMS.each do |metric_labels, value| + str << "http_request_duration_seconds_sum{" + str << "method=\"" << metric_labels.request_method << "\" " + str << "route=\"" << metric_labels.request_route << "\" " + str << "response_code=\"" << metric_labels.response_code << "\"" + str << "} " + str << value << "\n" + end + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS_PROMETHEUS.each.each do |key, value| + str << key << " " << value << "\n" + end + end end # APIv1 currently uses the same logic for both diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 27c3d199..47a2519d 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -311,6 +311,7 @@ module Invidious::Routing # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats if CONFIG.statistics_enabled + add_handler Metrics::METRICS_COLLECTOR get "/api/v1/metrics", {{namespace}}::Misc, :metrics end get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist From 8456f8d4cd0b9ef8a0a41aa153bdc21294e1c7eb Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Thu, 20 Jul 2023 19:36:22 +0200 Subject: [PATCH 3/7] Move metrics handler registration together with the rest of handlers, into invidious.cr --- src/invidious.cr | 1 + src/invidious/routing.cr | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..66a93bab 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -170,6 +170,7 @@ end if CONFIG.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) + add_handler Metrics::METRICS_COLLECTOR end if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 47a2519d..27c3d199 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -311,7 +311,6 @@ module Invidious::Routing # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats if CONFIG.statistics_enabled - add_handler Metrics::METRICS_COLLECTOR get "/api/v1/metrics", {{namespace}}::Misc, :metrics end get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist From e7f7f39ce851c3a243f0f92ed4526c2320049284 Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Thu, 20 Jul 2023 19:40:43 +0200 Subject: [PATCH 4/7] Return empty response on /api/v1/metrics endpoint if metrics are not enabled --- src/invidious/routing.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 27c3d199..5ee5ed58 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -312,6 +312,10 @@ module Invidious::Routing get "/api/v1/stats", {{namespace}}::Misc, :stats if CONFIG.statistics_enabled get "/api/v1/metrics", {{namespace}}::Misc, :metrics + else + get "/api/v1/metrics" do |env| + env.response.status_code = 204 + end end get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist From 9db6eb058cb55e1c25ced81ce064b7eacb57fa78 Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Thu, 20 Jul 2023 19:42:33 +0200 Subject: [PATCH 5/7] Fix formatting --- src/invidious/metrics.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/metrics.cr b/src/invidious/metrics.cr index 8007ca7d..7165b14f 100644 --- a/src/invidious/metrics.cr +++ b/src/invidious/metrics.cr @@ -4,12 +4,12 @@ module Metrics record MetricLabels, request_method : String, request_route : String, response_code : Int32 # Counts how many a given route was used - REQUEST_COUNTERS = Hash(MetricLabels, Int64).new + REQUEST_COUNTERS = Hash(MetricLabels, Int64).new # Counts how much time was used to handle requests to each route REQUEST_DURATION_SECONDS_SUMS = Hash(MetricLabels, Float32).new - # The handler which will record metrics when registered in a Kemal application + # The handler which will record metrics when registered in a Kemal application METRICS_COLLECTOR = RouteMetricsCollector.new(REQUEST_COUNTERS, REQUEST_DURATION_SECONDS_SUMS) class RouteMetricsCollector < Kemal::Handler From 8d4c16c79c4af2138ad791130e763ce128606d64 Mon Sep 17 00:00:00 2001 From: wint3rmute Date: Thu, 20 Jul 2023 20:16:15 +0200 Subject: [PATCH 6/7] Moved from misused constants to class variables in MetricsCollector, MetricsCollector is no longer initialized as a singleton --- src/invidious.cr | 2 +- src/invidious/metrics.cr | 36 ++++++++++++++--------------- src/invidious/routes/api/v1/misc.cr | 4 ++-- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 66a93bab..67fa0498 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -170,7 +170,7 @@ end if CONFIG.statistics_enabled Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, SOFTWARE) - add_handler Metrics::METRICS_COLLECTOR + add_handler Metrics::RouteMetricsCollector.new end if (CONFIG.use_pubsub_feeds.is_a?(Bool) && CONFIG.use_pubsub_feeds.as(Bool)) || (CONFIG.use_pubsub_feeds.is_a?(Int32) && CONFIG.use_pubsub_feeds.as(Int32) > 0) diff --git a/src/invidious/metrics.cr b/src/invidious/metrics.cr index 7165b14f..6fdd42b5 100644 --- a/src/invidious/metrics.cr +++ b/src/invidious/metrics.cr @@ -3,20 +3,18 @@ module Metrics record MetricLabels, request_method : String, request_route : String, response_code : Int32 - # Counts how many a given route was used - REQUEST_COUNTERS = Hash(MetricLabels, Int64).new - - # Counts how much time was used to handle requests to each route - REQUEST_DURATION_SECONDS_SUMS = Hash(MetricLabels, Float32).new - - # The handler which will record metrics when registered in a Kemal application - METRICS_COLLECTOR = RouteMetricsCollector.new(REQUEST_COUNTERS, REQUEST_DURATION_SECONDS_SUMS) - class RouteMetricsCollector < Kemal::Handler - def initialize( - @num_of_request_counters : Hash(MetricLabels, Int64), - @request_duration_seconds_sums : Hash(MetricLabels, Float32) - ) + # Counts how many times a given route was used + @@num_of_request_counters = Hash(MetricLabels, Int64).new + # Counts how much time was used to handle requests to each route + @@request_duration_seconds_sums = Hash(MetricLabels, Float32).new + + def self.num_of_request_counters + return @@num_of_request_counters + end + + def self.request_duration_seconds_sums + return @@request_duration_seconds_sums end def call(context : HTTP::Server::Context) @@ -33,15 +31,15 @@ module Metrics LOGGER.trace("Collecting metrics: handling #{request_method} #{request_path} took #{seconds_spent_handling}s and finished with status #{response_status}") metric_key = MetricLabels.new request_path, request_method, response_status - unless @num_of_request_counters.has_key?(metric_key) - @num_of_request_counters[metric_key] = 0 + unless @@num_of_request_counters.has_key?(metric_key) + @@num_of_request_counters[metric_key] = 0 end - @num_of_request_counters[metric_key] += 1 + @@num_of_request_counters[metric_key] += 1 - unless @request_duration_seconds_sums.has_key?(metric_key) - @request_duration_seconds_sums[metric_key] = 0.0 + unless @@request_duration_seconds_sums.has_key?(metric_key) + @@request_duration_seconds_sums[metric_key] = 0.0 end - @request_duration_seconds_sums[metric_key] += seconds_spent_handling + @@request_duration_seconds_sums[metric_key] += seconds_spent_handling end end end diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index d827b622..2a0ce5a7 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -32,7 +32,7 @@ module Invidious::Routes::API::V1::Misc env.response.content_type = "text/plain" return String.build do |str| - Metrics::REQUEST_COUNTERS.each do |metric_labels, value| + Metrics::RouteMetricsCollector.num_of_request_counters.each do |metric_labels, value| str << "http_requests_total{" str << "method=\"" << metric_labels.request_method << "\" " str << "route=\"" << metric_labels.request_route << "\" " @@ -41,7 +41,7 @@ module Invidious::Routes::API::V1::Misc str << value << "\n" end - Metrics::REQUEST_DURATION_SECONDS_SUMS.each do |metric_labels, value| + Metrics::RouteMetricsCollector.request_duration_seconds_sums.each do |metric_labels, value| str << "http_request_duration_seconds_sum{" str << "method=\"" << metric_labels.request_method << "\" " str << "route=\"" << metric_labels.request_route << "\" " From 70754659e5e3950c168c74473cc38a9499dc57f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20B=C4=85czek?= Date: Tue, 23 Jan 2024 21:24:08 +0100 Subject: [PATCH 7/7] Move `if CONFIG.statistics_enabled` into the handler for the /metrics route --- src/invidious/routes/api/v1/misc.cr | 5 +++++ src/invidious/routing.cr | 8 +------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/invidious/routes/api/v1/misc.cr b/src/invidious/routes/api/v1/misc.cr index 2a0ce5a7..886aa481 100644 --- a/src/invidious/routes/api/v1/misc.cr +++ b/src/invidious/routes/api/v1/misc.cr @@ -29,6 +29,11 @@ module Invidious::Routes::API::V1::Misc end def self.metrics(env) + if !CONFIG.statistics_enabled + env.response.status_code = 204 + return + end + env.response.content_type = "text/plain" return String.build do |str| diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 5ee5ed58..f89a0df9 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -310,13 +310,7 @@ module Invidious::Routing # Misc get "/api/v1/stats", {{namespace}}::Misc, :stats - if CONFIG.statistics_enabled - get "/api/v1/metrics", {{namespace}}::Misc, :metrics - else - get "/api/v1/metrics" do |env| - env.response.status_code = 204 - end - end + get "/api/v1/metrics", {{namespace}}::Misc, :metrics get "/api/v1/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/auth/playlists/:plid", {{namespace}}::Misc, :get_playlist get "/api/v1/mixes/:rdid", {{namespace}}::Misc, :mixes