class RedditThing include JSON::Serializable property kind : String property data : RedditComment | RedditLink | RedditMore | RedditListing end class RedditComment include JSON::Serializable property author : String property body_html : String property replies : RedditThing | String property score : Int32 property depth : Int32 property permalink : String @[JSON::Field(converter: RedditComment::TimeConverter)] property created_utc : Time module TimeConverter def self.from_json(value : JSON::PullParser) : Time Time.unix(value.read_float.to_i) end def self.to_json(value : Time, json : JSON::Builder) json.number(value.to_unix) end end end struct RedditLink include JSON::Serializable property author : String property score : Int32 property subreddit : String property num_comments : Int32 property id : String property permalink : String property title : String end struct RedditMore include JSON::Serializable property children : Array(String) property count : Int32 property depth : Int32 end class RedditListing include JSON::Serializable property children : Array(RedditThing) property modhash : String end def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_by = "top") case cursor when nil, "" ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by) # when .starts_with? "Ug" # ctoken = produce_comment_reply_continuation(id, video.ucid, cursor) when .starts_with? "ADSJ" ctoken = produce_comment_continuation(id, cursor: cursor, sort_by: sort_by) else ctoken = cursor end client_config = YoutubeAPI::ClientConfig.new(region: region) response = YoutubeAPI.next(continuation: ctoken, client_config: client_config) contents = nil if response["onResponseReceivedEndpoints"]? onResponseReceivedEndpoints = response["onResponseReceivedEndpoints"] header = nil onResponseReceivedEndpoints.as_a.each do |item| if item["reloadContinuationItemsCommand"]? case item["reloadContinuationItemsCommand"]["slot"] when "RELOAD_CONTINUATION_SLOT_HEADER" header = item["reloadContinuationItemsCommand"]["continuationItems"][0] when "RELOAD_CONTINUATION_SLOT_BODY" contents = item["reloadContinuationItemsCommand"]["continuationItems"] end elsif item["appendContinuationItemsAction"]? contents = item["appendContinuationItemsAction"]["continuationItems"] end end elsif response["continuationContents"]? response = response["continuationContents"] if response["commentRepliesContinuation"]? body = response["commentRepliesContinuation"] else body = response["itemSectionContinuation"] end contents = body["contents"]? header = body["header"]? if body["continuations"]? moreRepliesContinuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s end else raise InfoException.new("Could not fetch comments") end if !contents if format == "json" return {"comments" => [] of String}.to_json else return {"contentHtml" => "", "commentCount" => 0}.to_json end end continuationItemRenderer = nil contents.as_a.reject! do |item| if item["continuationItemRenderer"]? continuationItemRenderer = item["continuationItemRenderer"] true end end response = JSON.build do |json| json.object do if header count_text = header["commentsHeaderRenderer"]["countText"] comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s.gsub(/\D/, "").to_i? || 0 json.field "commentCount", comment_count end json.field "videoId", id json.field "comments" do json.array do contents.as_a.each do |node| json.object do if node["commentThreadRenderer"]? node = node["commentThreadRenderer"] end if node["replies"]? node_replies = node["replies"]["commentRepliesRenderer"] end if node["comment"]? node_comment = node["comment"]["commentRenderer"] else node_comment = node["commentRenderer"] end content_html = node_comment["contentText"]?.try { |t| parse_content(t) } || "" author = node_comment["authorText"]?.try &.["simpleText"]? || "" json.field "author", author json.field "authorThumbnails" do json.array do node_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| json.object do json.field "url", thumbnail["url"] json.field "width", thumbnail["width"] json.field "height", thumbnail["height"] end end end end if node_comment["authorEndpoint"]? json.field "authorId", node_comment["authorEndpoint"]["browseEndpoint"]["browseId"] json.field "authorUrl", node_comment["authorEndpoint"]["browseEndpoint"]["canonicalBaseUrl"] else json.field "authorId", "" json.field "authorUrl", "" end published_text = node_comment["publishedTimeText"]["runs"][0]["text"].as_s published = decode_date(published_text.rchop(" (edited)")) if published_text.includes?(" (edited)") json.field "isEdited", true else json.field "isEdited", false end json.field "content", html_to_content(content_html) json.field "contentHtml", content_html json.field "published", published.to_unix json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale)) comment_action_buttons_renderer = node_comment["actionButtons"]["commentActionButtonsRenderer"] json.field "likeCount", comment_action_buttons_renderer["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"].as_s.scan(/\d/).map(&.[0]).join.to_i json.field "commentId", node_comment["commentId"] json.field "authorIsChannelOwner", node_comment["authorIsChannelOwner"] if comment_action_buttons_renderer["creatorHeart"]? hearth_data = comment_action_buttons_renderer["creatorHeart"]["creatorHeartRenderer"]["creatorThumbnail"] json.field "creatorHeart" do json.object do json.field "creatorThumbnail", hearth_data["thumbnails"][-1]["url"] json.field "creatorName", hearth_data["accessibility"]["accessibilityData"]["label"] end end end if node_replies && !response["commentRepliesContinuation"]? if node_replies["moreText"]? reply_count = (node_replies["moreText"]["simpleText"]? || node_replies["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) .try &.as_s.gsub(/\D/, "").to_i? || 1 elsif node_replies["viewReplies"]? reply_count = node_replies["viewReplies"]["buttonRenderer"]["text"]?.try &.["runs"][1]?.try &.["text"]?.try &.as_s.to_i? || 1 else reply_count = 1 end if node_replies["continuations"]? continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s elsif node_replies["contents"]? continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s end continuation ||= "" json.field "replies" do json.object do json.field "replyCount", reply_count json.field "continuation", continuation end end end end end end end if continuationItemRenderer if continuationItemRenderer["continuationEndpoint"]? continuationEndpoint = continuationItemRenderer["continuationEndpoint"] elsif continuationItemRenderer["button"]? continuationEndpoint = continuationItemRenderer["button"]["buttonRenderer"]["command"] end if continuationEndpoint json.field "continuation", continuationEndpoint["continuationCommand"]["token"].as_s end end end end if format == "html" response = JSON.parse(response) content_html = template_youtube_comments(response, locale, thin_mode) response = JSON.build do |json| json.object do json.field "contentHtml", content_html if response["commentCount"]? json.field "commentCount", response["commentCount"] else json.field "commentCount", 0 end end end end return response end def fetch_reddit_comments(id, sort_by = "confidence") client = make_client(REDDIT_URL) headers = HTTP::Headers{"User-Agent" => "web:invidious:v#{CURRENT_VERSION} (by github.com/iv-org/invidious)"} # TODO: Use something like #479 for a static list of instances to use here query = "(url:3D#{id}%20OR%20url:#{id})%20(site:invidio.us%20OR%20site:youtube.com%20OR%20site:youtu.be)" search_results = client.get("/search.json?q=#{query}", headers) if search_results.status_code == 200 search_results = RedditThing.from_json(search_results.body) # For videos that have more than one thread, choose the one with the highest score thread = search_results.data.as(RedditListing).children.sort_by { |child| child.data.as(RedditLink).score }[-1] thread = thread.data.as(RedditLink) result = client.get("/r/#{thread.subreddit}/comments/#{thread.id}.json?limit=100&sort=#{sort_by}", headers).body result = Array(RedditThing).from_json(result) elsif search_results.status_code == 302 # Previously, if there was only one result then the API would redirect to that result. # Now, it appears it will still return a listing so this section is likely unnecessary. result = client.get(search_results.headers["Location"], headers).body result = Array(RedditThing).from_json(result) thread = result[0].data.as(RedditListing).children[0].data.as(RedditLink) else raise InfoException.new("Could not fetch comments") end client.close comments = result[1].data.as(RedditListing).children return comments, thread end def template_youtube_comments(comments, locale, thin_mode, is_replies = false) String.build do |html| root = comments["comments"].as_a root.each do |child| if child["replies"]? replies_html = <<-END_HTML
END_HTML end if !thin_mode author_thumbnail = "/ggpht#{URI.parse(child["authorThumbnails"][-1]["url"].as_s).request_target}" else author_thumbnail = "" end author_name = HTML.escape(child["author"].as_s) html << <<-END_HTML#{child["contentHtml"]}
END_HTML if child["attachment"]? attachment = child["attachment"] case attachment["type"] when "image" attachment = attachment["imageThumbnails"][1] html << <<-END_HTML#{attachment["error"]}
END_HTML else html << <<-END_HTML END_HTML end html << <<-END_HTML[ - ] #{child.author} #{translate(locale, "`x` points", number_with_separator(child.score))} #{translate(locale, "`x` ago", recode_date(child.created_utc, locale))} #{translate(locale, "permalink")}