From 2d6748aae40a99cd0cc94c5542bb628bfd8afb6d Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Mon, 18 Dec 2023 01:19:40 -0800 Subject: [PATCH 1/6] Add migrate script --- src/invidious.cr | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..6becc055 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -138,6 +138,9 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity Invidious::Database.check_integrity(CONFIG) +# Run migration +Invidious::Database::Migrator.new(PG_DB).migrate + {% if !flag?(:skip_videojs_download) %} # Resolve player dependencies. This is done at compile time. # From acf0c19fa96302158603d21ed4aa66a628e82651 Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Thu, 21 Dec 2023 02:23:38 -0800 Subject: [PATCH 2/6] Add backup creation --- docker/init-invidious-db.sh | 2 ++ src/invidious/database/migrator.cr | 39 +++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/docker/init-invidious-db.sh b/docker/init-invidious-db.sh index 22b4cc5f..afdc001e 100755 --- a/docker/init-invidious-db.sh +++ b/docker/init-invidious-db.sh @@ -1,6 +1,8 @@ #!/bin/bash set -eou pipefail +psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" -c "CREATE SCHEMA IF NOT EXISTS backup" + psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channels.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/videos.sql psql --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < config/sql/channel_videos.sql diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr index 660c3203..53260d3f 100644 --- a/src/invidious/database/migrator.cr +++ b/src/invidious/database/migrator.cr @@ -10,10 +10,18 @@ class Invidious::Database::Migrator versions = load_versions ran_migration = false + backed_up = false load_migrations.sort_by(&.version) .each do |migration| next if versions.includes?(migration.version) + + if !backed_up + puts "New migration(s) found: creating database backup" + back_up_database + backed_up = true + end + puts "Running migration: #{migration.class.name}" migration.migrate ran_migration = true @@ -46,4 +54,33 @@ class Invidious::Database::Migrator ) SQL end -end + + private def back_up_database + table_names_request = <<-SQL + SELECT tablename FROM pg_catalog.pg_tables + WHERE schemaname = 'public' + SQL + + table_names = @db.query_all(table_names_request, as: String) + + table_names.try &.each do |name| + copy_table(name) + end + end + + private def copy_table(table_name : String) + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS backup.#{table_name} ( + id bigserial PRIMARY KEY + ) + SQL + + @db.exec("DROP TABLE backup.#{table_name}") + + @db.exec <<-SQL + SELECT * INTO backup.#{table_name} + FROM public.#{table_name} + SQL + end + +end \ No newline at end of file From 559d3e32c4a134bd70cbf65f6deb1e0687a1527a Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Thu, 21 Dec 2023 02:25:32 -0800 Subject: [PATCH 3/6] Remove auto migrate line --- src/invidious.cr | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 6becc055..e0bd0101 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -138,9 +138,6 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Check table integrity Invidious::Database.check_integrity(CONFIG) -# Run migration -Invidious::Database::Migrator.new(PG_DB).migrate - {% if !flag?(:skip_videojs_download) %} # Resolve player dependencies. This is done at compile time. # From 4eee5cf70f07777398756bfad77bceefa4868f55 Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Thu, 21 Dec 2023 18:14:40 -0800 Subject: [PATCH 4/6] Run crystal format --- src/invidious/database/migrator.cr | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr index 53260d3f..41340253 100644 --- a/src/invidious/database/migrator.cr +++ b/src/invidious/database/migrator.cr @@ -15,12 +15,11 @@ class Invidious::Database::Migrator .each do |migration| next if versions.includes?(migration.version) - if !backed_up puts "New migration(s) found: creating database backup" back_up_database backed_up = true - end + end puts "Running migration: #{migration.class.name}" migration.migrate @@ -60,13 +59,13 @@ class Invidious::Database::Migrator SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = 'public' SQL - + table_names = @db.query_all(table_names_request, as: String) table_names.try &.each do |name| copy_table(name) - end - end + end + end private def copy_table(table_name : String) @db.exec <<-SQL @@ -76,11 +75,10 @@ class Invidious::Database::Migrator SQL @db.exec("DROP TABLE backup.#{table_name}") - + @db.exec <<-SQL SELECT * INTO backup.#{table_name} FROM public.#{table_name} SQL - end - -end \ No newline at end of file + end +end From aef8215402c245e69d49c39e4b9cb76501f92938 Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Sat, 3 Aug 2024 18:01:52 -0700 Subject: [PATCH 5/6] Only drop the second most recent backup --- src/invidious/database/migrator.cr | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr index 41340253..1ec7b6ca 100644 --- a/src/invidious/database/migrator.cr +++ b/src/invidious/database/migrator.cr @@ -68,17 +68,32 @@ class Invidious::Database::Migrator end private def copy_table(table_name : String) + puts "Creating a table backup.#{table_name}. Most recent version" @db.exec <<-SQL CREATE TABLE IF NOT EXISTS backup.#{table_name} ( id bigserial PRIMARY KEY ) SQL - @db.exec("DROP TABLE backup.#{table_name}") + puts "Creating a table backup.#{table_name}. Second most recent version" + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS backup.#{table_name}_second_most_recent ( + id bigserial PRIMARY KEY + ) + SQL + + puts "Populating table backup.#{table_name}. Second most recent version" + @db.exec <<-SQL + SELECT * INTO backup.#{table_name}_second_most_recent + FROM backup.#{table_name} + SQL @db.exec <<-SQL SELECT * INTO backup.#{table_name} FROM public.#{table_name} SQL + + puts "Deleting table backup.#{table_name}. Second most recent version" + @db.exec("DROP TABLE backup.#{table_name}_second_most_recent") end end From d806d1f41dd02e90a4a980ccda17868a3ab01bab Mon Sep 17 00:00:00 2001 From: broquemonsieur Date: Sun, 4 Aug 2024 02:13:07 -0700 Subject: [PATCH 6/6] Add log message --- src/invidious/database/migrator.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr index 1ec7b6ca..de508346 100644 --- a/src/invidious/database/migrator.cr +++ b/src/invidious/database/migrator.cr @@ -88,6 +88,7 @@ class Invidious::Database::Migrator FROM backup.#{table_name} SQL + puts "Populating table backup.#{table_name}. Most recent version" @db.exec <<-SQL SELECT * INTO backup.#{table_name} FROM public.#{table_name}