From 1e2d8623e27f0f31941ba490a95d99879922204d Mon Sep 17 00:00:00 2001 From: fkrueger Date: Sun, 7 Jan 2024 02:53:25 +0100 Subject: [PATCH 1/4] FK: created a workaround for this 2+ months old problem import googles export / takeout of youtube playlists. not perfect (ie. things like "providing the takeout zip and all playlists get imported".. not yet supported). --- src/invidious/user/imports.cr | 82 +++++++++++++++++++++++++++++------ 1 file changed, 68 insertions(+), 14 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 108f2ccc..8beaf018 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,21 +30,75 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, raw_input : String) + + def parse_playlist_export_csv(user : User, raw_input : String, filename : String) + + LOGGER.trace("parse_playlist_export_csv: 01 raw_input '#{raw_input}'\n") + + raw_head = "" # playlists.csv - info-line for a given playlist in the google export, that's copied above the actual playlist-infos and separated by an empty line from + raw_body = "" # the actual playlist content, ie. a list of videos + + # remove superfluous \n instances in front of and after any non-\n text.. so .. a trim? + raw_input = raw_input.strip('\n') + # Split the input into head and body content - raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true) - - # Create the playlist from the head content - csv_head = CSV.new(raw_head.strip('\n'), headers: true) - csv_head.next - title = csv_head[4] - description = csv_head[5] - visibility = csv_head[6] - - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy::Public + tmp = raw_input.split("\n\n", limit: 2, remove_empty: true) + if tmp.size > 1 + raw_head = tmp[0] + raw_body = tmp[1] else - privacy = PlaylistPrivacy::Private + raw_body = tmp[0] + end + + LOGGER.trace("parse_playlist_export_csv: 02 raw_head '#{raw_head}' -----\nraw_body '#{raw_body}'\n") + + # TODO create an import-feature (elsewhere), which works for the original google export format, ie. the playlists/ subdirectory content as a ZIP file.. or a complete youtube music export file, but this goes way beyond the scope of a playlist import. + + # defaults + title = "title not set" + description = "from #{filename}" + visibility = "Private" + privacy = PlaylistPrivacy::Private + + if (raw_head != "") + + # XXX in 2023/2024 this looked like this: + #0 1 2 3 4 5 6 7 8 9 10 11 + #Playlist ID,Add new videos to top,Playlist image 1 Create timestamp,Playlist image 1 URL,Playlist image 1 Height,Playlist image 1 Width,Playlist title (original),Playlist title (original) language,Playlist create timestamp,Playlist update timestamp,Playlist video order,Playlist visibility + #PLc5oiabcabcabcabcabcrOvXgzabcabcm,False,,,,,display name of playlist,,2015-01-01T01:02:03+00:00,2022-10-28T02:23:15+00:00,Manual,Public' --- + + # Create the playlist from the head content + csv_head = CSV.new(raw_head.strip('\n'), headers: true) + csv_head.next + if csv_head[11] + LOGGER.info("parse_playlist_export_csv: 03.1 raw_head is filled, doing dual csv playlist info scan. this seems to be one line out of the google export's playlists.csv file, but google never adds these to the several playlist files in an export.") + title = csv_head[6] + description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" + visibility = csv_head[11] + else + LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is filled, doing old default behaviour.") # XXX no idea why though. + title = csv_head[4] + description = csv_head[5] + visibility = csv_head[6] + end + + if visibility.compare("Public", case_insensitive: true) == 0 + privacy = PlaylistPrivacy::Public + else + privacy = PlaylistPrivacy::Private + end + else # choose a title from the provided upload filename + LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is empty. Trying to guess fine-looking title and description from the provided upload filename.") + if (filename != "") + title = filename + end + nameendpos = filename.rindex(" videos.", filename.size) # XXX everything up to " videos.csv" + if !nameendpos # XXX everything up to file extension + nameendpos = filename.rindex(".", filename.size) + end + if nameendpos + title = filename[0, nameendpos] + end end playlist = create_playlist(title, privacy, user) @@ -207,7 +261,7 @@ struct Invidious::User extension = filename.split(".").last if extension == "csv" || type == "text/csv" - playlist = parse_playlist_export_csv(user, body) + playlist = parse_playlist_export_csv(user, body, filename) if playlist return true else From 7775f46fc6b74439d5af519f236db4c342ca42dc Mon Sep 17 00:00:00 2001 From: fkrueger Date: Sun, 7 Jan 2024 12:25:57 +0100 Subject: [PATCH 2/4] FK: cleaned up the "logic" and added a few tags for future reference --- src/invidious/user/imports.cr | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 8beaf018..73ca41fe 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -55,31 +55,34 @@ struct Invidious::User # TODO create an import-feature (elsewhere), which works for the original google export format, ie. the playlists/ subdirectory content as a ZIP file.. or a complete youtube music export file, but this goes way beyond the scope of a playlist import. # defaults - title = "title not set" - description = "from #{filename}" + title = "title not set" # TODO i8n + description = "from #{filename}" # TODO i8n visibility = "Private" privacy = PlaylistPrivacy::Private if (raw_head != "") - # XXX in 2023/2024 this looked like this: + ### XXX for documentation of current file format of playlists.csv (today it's 2024-01-07): #0 1 2 3 4 5 6 7 8 9 10 11 #Playlist ID,Add new videos to top,Playlist image 1 Create timestamp,Playlist image 1 URL,Playlist image 1 Height,Playlist image 1 Width,Playlist title (original),Playlist title (original) language,Playlist create timestamp,Playlist update timestamp,Playlist video order,Playlist visibility #PLc5oiabcabcabcabcabcrOvXgzabcabcm,False,,,,,display name of playlist,,2015-01-01T01:02:03+00:00,2022-10-28T02:23:15+00:00,Manual,Public' --- - - # Create the playlist from the head content + ###/documentation + + # Create the playlist from the head content (if it seems to be valid): csv_head = CSV.new(raw_head.strip('\n'), headers: true) csv_head.next if csv_head[11] - LOGGER.info("parse_playlist_export_csv: 03.1 raw_head is filled, doing dual csv playlist info scan. this seems to be one line out of the google export's playlists.csv file, but google never adds these to the several playlist files in an export.") + LOGGER.info("parse_playlist_export_csv: 03.1 raw_head is filled, doing dual csv playlist info scan. google takeout format after october 2023 (format seems to be one line of playlist metadata taken out of the google export's playlists.csv file, added above the actual playlist.csv content, separated by an empty line)") title = csv_head[6] - description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" + description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" # TODO i8n visibility = csv_head[11] - else - LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is filled, doing old default behaviour.") # XXX no idea why though. + else if csv_head[6] + LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is filled, doing dual csv playlist info scan. google takeout format before october 2023 (roughly).") title = csv_head[4] description = csv_head[5] visibility = csv_head[6] + else # we are using the defaults defined above instead. + LOGGER.info("parse_playlist_export_csv: 03.3 raw_head is filled, but in an unknown format. Using base defaults instead.") end if visibility.compare("Public", case_insensitive: true) == 0 @@ -110,7 +113,7 @@ struct Invidious::User video_id = row[0] if playlist next if !video_id - next if video_id == "Video Id" + next if video_id == "Video Id" # TODO i8n begin video = get_video(video_id) From e5655008ccb01b930fd49ff821254adc3bd9d875 Mon Sep 17 00:00:00 2001 From: fkrueger Date: Sun, 7 Jan 2024 12:32:00 +0100 Subject: [PATCH 3/4] FK: I m new to crystal, ok? I made the error to trust the first official looking crystal (reports) documentation that showed up on google. perl-syntax for elsif definitely is preferrable to the basic BS of else if :-) --- src/invidious/user/imports.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 73ca41fe..fcb565dc 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -76,7 +76,7 @@ struct Invidious::User title = csv_head[6] description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" # TODO i8n visibility = csv_head[11] - else if csv_head[6] + elsif csv_head[6] LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is filled, doing dual csv playlist info scan. google takeout format before october 2023 (roughly).") title = csv_head[4] description = csv_head[5] From 4a70164539b423ff1f880664b18a9f058c413e47 Mon Sep 17 00:00:00 2001 From: fkrueger Date: Fri, 10 May 2024 13:22:24 +0200 Subject: [PATCH 4/4] FK: let crystal tool format do its magic and replace the spaces before in-line comments with a random number of tabs, to get the lint check to work --- src/invidious/user/imports.cr | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index fcb565dc..8aa5273f 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,13 +30,11 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, raw_input : String, filename : String) - LOGGER.trace("parse_playlist_export_csv: 01 raw_input '#{raw_input}'\n") - raw_head = "" # playlists.csv - info-line for a given playlist in the google export, that's copied above the actual playlist-infos and separated by an empty line from - raw_body = "" # the actual playlist content, ie. a list of videos + raw_head = "" # playlists.csv - info-line for a given playlist in the google export, that's copied above the actual playlist-infos and separated by an empty line from + raw_body = "" # the actual playlist content, ie. a list of videos # remove superfluous \n instances in front of and after any non-\n text.. so .. a trim? raw_input = raw_input.strip('\n') @@ -55,18 +53,17 @@ struct Invidious::User # TODO create an import-feature (elsewhere), which works for the original google export format, ie. the playlists/ subdirectory content as a ZIP file.. or a complete youtube music export file, but this goes way beyond the scope of a playlist import. # defaults - title = "title not set" # TODO i8n - description = "from #{filename}" # TODO i8n + title = "title not set" # TODO i8n + description = "from #{filename}" # TODO i8n visibility = "Private" privacy = PlaylistPrivacy::Private if (raw_head != "") - - ### XXX for documentation of current file format of playlists.csv (today it's 2024-01-07): - #0 1 2 3 4 5 6 7 8 9 10 11 - #Playlist ID,Add new videos to top,Playlist image 1 Create timestamp,Playlist image 1 URL,Playlist image 1 Height,Playlist image 1 Width,Playlist title (original),Playlist title (original) language,Playlist create timestamp,Playlist update timestamp,Playlist video order,Playlist visibility - #PLc5oiabcabcabcabcabcrOvXgzabcabcm,False,,,,,display name of playlist,,2015-01-01T01:02:03+00:00,2022-10-28T02:23:15+00:00,Manual,Public' --- - ###/documentation + # ## XXX for documentation of current file format of playlists.csv (today it's 2024-01-07): + # 0 1 2 3 4 5 6 7 8 9 10 11 + # Playlist ID,Add new videos to top,Playlist image 1 Create timestamp,Playlist image 1 URL,Playlist image 1 Height,Playlist image 1 Width,Playlist title (original),Playlist title (original) language,Playlist create timestamp,Playlist update timestamp,Playlist video order,Playlist visibility + # PLc5oiabcabcabcabcabcrOvXgzabcabcm,False,,,,,display name of playlist,,2015-01-01T01:02:03+00:00,2022-10-28T02:23:15+00:00,Manual,Public' --- + # ##/documentation # Create the playlist from the head content (if it seems to be valid): csv_head = CSV.new(raw_head.strip('\n'), headers: true) @@ -74,14 +71,14 @@ struct Invidious::User if csv_head[11] LOGGER.info("parse_playlist_export_csv: 03.1 raw_head is filled, doing dual csv playlist info scan. google takeout format after october 2023 (format seems to be one line of playlist metadata taken out of the google export's playlists.csv file, added above the actual playlist.csv content, separated by an empty line)") title = csv_head[6] - description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" # TODO i8n + description = "Playlist was imported from file '#{filename}'\n\nCreated on #{csv_head[8]}\nLast updated on #{csv_head[9]}\n" # TODO i8n visibility = csv_head[11] elsif csv_head[6] LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is filled, doing dual csv playlist info scan. google takeout format before october 2023 (roughly).") title = csv_head[4] description = csv_head[5] visibility = csv_head[6] - else # we are using the defaults defined above instead. + else # we are using the defaults defined above instead. LOGGER.info("parse_playlist_export_csv: 03.3 raw_head is filled, but in an unknown format. Using base defaults instead.") end @@ -90,13 +87,13 @@ struct Invidious::User else privacy = PlaylistPrivacy::Private end - else # choose a title from the provided upload filename + else # choose a title from the provided upload filename LOGGER.info("parse_playlist_export_csv: 03.2 raw_head is empty. Trying to guess fine-looking title and description from the provided upload filename.") if (filename != "") title = filename end - nameendpos = filename.rindex(" videos.", filename.size) # XXX everything up to " videos.csv" - if !nameendpos # XXX everything up to file extension + nameendpos = filename.rindex(" videos.", filename.size) # XXX everything up to " videos.csv" + if !nameendpos # XXX everything up to file extension nameendpos = filename.rindex(".", filename.size) end if nameendpos @@ -113,7 +110,7 @@ struct Invidious::User video_id = row[0] if playlist next if !video_id - next if video_id == "Video Id" # TODO i8n + next if video_id == "Video Id" # TODO i8n begin video = get_video(video_id)