invidious-experimenting/src/invidious/playlists.cr

549 lines
16 KiB
Crystal
Raw Normal View History

2019-03-30 03:00:02 +05:30
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
2020-06-16 03:40:30 +05:30
def to_xml(auto_generated, xml : XML::Builder)
2019-08-06 05:19:13 +05:30
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
2020-06-16 03:40:30 +05:30
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
2019-08-06 05:19:13 +05:30
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
2020-06-16 03:40:30 +05:30
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
2019-08-06 05:19:13 +05:30
else
xml.element("name") { xml.text author }
2020-06-16 03:40:30 +05:30
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
2019-08-06 05:19:13 +05:30
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
2020-06-16 03:40:30 +05:30
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
2019-08-06 05:19:13 +05:30
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
2020-06-16 03:40:30 +05:30
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
2019-08-06 05:19:13 +05:30
width: "320", height: "180")
end
end
end
2020-06-16 03:40:30 +05:30
def to_xml(auto_generated, xml : XML::Builder? = nil)
2019-08-06 05:19:13 +05:30
if xml
2020-06-16 03:40:30 +05:30
to_xml(auto_generated, xml)
2019-08-06 05:19:13 +05:30
else
XML.build do |json|
2020-06-16 03:40:30 +05:30
to_xml(auto_generated, xml)
2019-08-06 05:19:13 +05:30
end
end
end
2020-06-16 03:40:30 +05:30
def to_json(locale, json : JSON::Builder, index : Int32?)
2019-06-09 00:01:41 +05:30
json.object do
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
2020-06-16 03:40:30 +05:30
generate_thumbnails(json, self.id)
2019-06-09 00:01:41 +05:30
end
2019-08-06 05:19:13 +05:30
if index
json.field "index", index
json.field "indexId", self.index.to_u64.to_s(16).upcase
else
json.field "index", self.index
end
2019-06-09 00:01:41 +05:30
json.field "lengthSeconds", self.length_seconds
end
end
2020-06-16 03:40:30 +05:30
def to_json(locale, json : JSON::Builder? = nil, index : Int32? = nil)
2019-06-09 00:01:41 +05:30
if json
2020-06-16 03:40:30 +05:30
to_json(locale, json, index: index)
2019-06-09 00:01:41 +05:30
else
JSON.build do |json|
2020-06-16 03:40:30 +05:30
to_json(locale, json, index: index)
2019-06-09 00:01:41 +05:30
end
end
end
2018-09-29 09:42:35 +05:30
end
2019-03-30 03:00:02 +05:30
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
2020-06-16 03:40:30 +05:30
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
2019-08-06 05:19:13 +05:30
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
2019-08-06 05:19:13 +05:30
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
2020-06-16 03:40:30 +05:30
video.to_json(locale, json)
2019-08-06 05:19:13 +05:30
end
end
end
end
end
2020-06-16 03:40:30 +05:30
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
2019-08-06 05:19:13 +05:30
if json
2020-06-16 03:40:30 +05:30
to_json(offset, locale, json, continuation: continuation)
2019-08-06 05:19:13 +05:30
else
JSON.build do |json|
2020-06-16 03:40:30 +05:30
to_json(offset, locale, json, continuation: continuation)
2019-08-06 05:19:13 +05:30
end
end
end
def privacy
PlaylistPrivacy::Public
end
2018-08-15 20:52:36 +05:30
end
2019-08-06 05:19:13 +05:30
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
2019-08-06 05:19:13 +05:30
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
2020-06-16 03:40:30 +05:30
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
2019-08-06 05:19:13 +05:30
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
2018-10-08 07:41:33 +05:30
2019-08-06 05:19:13 +05:30
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if !offset || offset == 0
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64)
offset = self.index.index(index) || 0
end
2019-08-06 05:19:13 +05:30
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
2020-06-16 03:40:30 +05:30
video.to_json(locale, json, offset + index)
2019-08-06 05:19:13 +05:30
end
end
end
2018-10-08 07:41:33 +05:30
end
end
2020-06-16 03:40:30 +05:30
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
2019-08-06 05:19:13 +05:30
if json
2020-06-16 03:40:30 +05:30
to_json(offset, locale, json, continuation: continuation)
2019-08-06 05:19:13 +05:30
else
JSON.build do |json|
2020-06-16 03:40:30 +05:30
to_json(offset, locale, json, continuation: continuation)
2019-08-06 05:19:13 +05:30
end
end
end
2019-08-06 05:19:13 +05:30
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
2019-08-06 05:19:13 +05:30
def author_thumbnail
nil
2018-08-15 20:52:36 +05:30
end
2019-08-06 05:19:13 +05:30
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description).gsub("\n", "<br>")
end
end
def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
2019-08-06 05:19:13 +05:30
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
2019-08-06 05:19:13 +05:30
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
return playlist
end
def subscribe_playlist(db, user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title.byte_slice(0, 150),
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
playlist_array = playlist.to_a
args = arg_array(playlist_array)
db.exec("INSERT INTO playlists VALUES (#{args})", args: playlist_array)
return playlist
end
def produce_playlist_continuation(id, index)
2018-08-15 20:52:36 +05:30
if id.starts_with? "UC"
id = "UU" + id.lchop("UC")
end
2019-10-27 23:20:42 +05:30
plid = "VL" + id
# Emulate a "request counter" increment, to make perfectly valid
# ctokens, even if at the time of writing, it's ignored by youtube.
2021-03-21 20:35:50 +05:30
request_count = (index / 100).to_i64 || 1_i64
2019-10-27 23:20:42 +05:30
data = {"1:varint" => index.to_i64}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
2021-03-21 20:35:50 +05:30
data_wrapper = {"1:varint" => request_count, "15:string" => "PT:#{data}"}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
2019-10-27 23:20:42 +05:30
object = {
"80226972:embedded" => {
2021-03-21 20:35:50 +05:30
"2:string" => plid,
"3:string" => data_wrapper,
"35:string" => id,
2019-10-27 23:20:42 +05:30
},
}
continuation = object.try { |i| Protodec::Any.cast_json(object) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
2018-08-15 20:52:36 +05:30
end
2019-08-06 05:19:13 +05:30
def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
if plid.starts_with? "IV"
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist)
return playlist
else
raise InfoException.new("Playlist does not exist.")
2019-08-06 05:19:13 +05:30
end
else
return fetch_playlist(plid, locale)
end
end
2018-12-21 03:02:09 +05:30
def fetch_playlist(plid, locale)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
2018-08-15 20:52:36 +05:30
playlist_sidebar_renderer = initial_data["sidebar"]?.try &.["playlistSidebarRenderer"]?.try &.["items"]?
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer[0]["playlistSidebarPrimaryInfoRenderer"]?
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info["title"]?.try &.["runs"][0]?.try &.["text"]?.try &.as_s || ""
desc_item = playlist_info["description"]?
description_txt = desc_item.try &.["runs"]?.try &.as_a
.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info["thumbnailRenderer"]?.try &.["playlistVideoThumbnailRenderer"]?
.try &.["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s
2018-08-15 20:52:36 +05:30
views = 0_i64
updated = Time.utc
video_count = 0
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
2020-07-25 22:27:15 +05:30
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
2020-07-25 22:27:15 +05:30
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
else
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
2018-08-15 20:52:36 +05:30
if playlist_sidebar_renderer.size < 2
author = ""
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1]["playlistSidebarSecondaryInfoRenderer"]?.try &.["videoOwner"]["videoOwnerRenderer"]?
raise InfoException.new("Could not extract author info") if !author_info
2019-05-01 18:33:58 +05:30
author = author_info["title"]["runs"][0]["text"]?.try &.as_s || ""
author_thumbnail = author_info["thumbnail"]["thumbnails"][0]["url"]?.try &.as_s || ""
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
end
2018-08-15 20:52:36 +05:30
return Playlist.new({
title: title,
id: plid,
author: author,
2018-12-16 00:32:53 +05:30
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description_txt,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
})
2018-08-15 20:52:36 +05:30
end
2018-10-08 07:41:33 +05:30
2019-08-06 05:19:13 +05:30
def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
# Show empy playlist if requested page is out of range
2021-04-07 18:43:41 +05:30
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
2019-08-06 05:19:13 +05:30
end
if playlist.is_a? InvidiousPlaylist
db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3",
playlist.id, playlist.index, offset, as: PlaylistVideo)
else
if offset >= 100
# Normalize offset to match youtube's behavior (100 videos chunck per request)
offset = (offset / 100).to_i64 * 100_i64
2019-08-06 05:19:13 +05:30
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = YoutubeAPI.browse(ctoken)
else
initial_data = YoutubeAPI.browse("VL" + playlist.id, params: "")
end
return extract_playlist_videos(initial_data)
2019-08-06 05:19:13 +05:30
end
end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]
# Watch out the two versions, with and without "s"
if tabs_renderer["contents"]? || tabs_renderer["content"]?
# Initial playlist data
tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"]
list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0]
item_renderer = list_renderer.["itemSectionRenderer"]["contents"][0]
contents = item_renderer.["playlistVideoListRenderer"]["contents"].as_a
else
# Continuation data
contents = initial_data["onResponseReceivedActions"][0]?
.try &.["appendContinuationItemsAction"]["continuationItems"].as_a
end
else
contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a
end
contents.try &.each do |item|
if i = item["playlistVideoRenderer"]?
video_id = i["navigationEndpoint"]["watchEndpoint"]["videoId"].as_s
plid = i["navigationEndpoint"]["watchEndpoint"]["playlistId"].as_s
index = i["navigationEndpoint"]["watchEndpoint"]["index"].as_i64
thumbnail = i["thumbnail"]["thumbnails"][0]["url"].as_s
title = i["title"].try { |t| t["simpleText"]? || t["runs"]?.try &.[0]["text"]? }.try &.as_s || ""
author = i["shortBylineText"]?.try &.["runs"][0]["text"].as_s || ""
ucid = i["shortBylineText"]?.try &.["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"].as_s || ""
length_seconds = i["lengthSeconds"]?.try &.as_s.to_i
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title,
id: video_id,
author: author,
ucid: ucid,
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index,
})
end
end
return videos
end
2018-10-08 07:41:33 +05:30
def template_playlist(playlist)
html = <<-END_HTML
<h3>
<a href="/playlist?list=#{playlist["playlistId"]}">
#{playlist["title"]}
</a>
</h3>
<div class="pure-menu pure-menu-scrollable playlist-restricted">
<ol class="pure-menu-list">
END_HTML
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}">
<div class="thumbnail">
<img class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg">
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
2018-10-08 07:41:33 +05:30
<p style="width:100%">#{video["title"]}</p>
<p>
2019-05-02 06:33:39 +05:30
<b style="width:100%">#{video["author"]}</b>
2018-10-08 07:41:33 +05:30
</p>
</a>
</li>
END_HTML
end
html += <<-END_HTML
</ol>
</div>
<hr>
END_HTML
html
end