diff --git a/invidious.sql b/invidious.sql deleted file mode 100644 index fb8c687f..00000000 --- a/invidious.sql +++ /dev/null @@ -1,37 +0,0 @@ --- Table: public.videos - --- DROP TABLE public.videos; - -CREATE TABLE public.videos -( - last_updated timestamp with time zone, - video_id text COLLATE pg_catalog."default" NOT NULL, - video_info text COLLATE pg_catalog."default", - video_html text COLLATE pg_catalog."default", - views bigint, - likes integer, - dislikes integer, - rating double precision, - description text COLLATE pg_catalog."default", - CONSTRAINT videos_pkey PRIMARY KEY (video_id) -) -WITH ( - OIDS = FALSE -) -TABLESPACE pg_default; - -ALTER TABLE public.videos - OWNER to omar; - -GRANT ALL ON TABLE public.videos TO kemal; - -GRANT ALL ON TABLE public.videos TO omar; - --- Index: videos_video_id_idx - --- DROP INDEX public.videos_video_id_idx; - -CREATE INDEX videos_video_id_idx - ON public.videos USING btree - (video_id COLLATE pg_catalog."default") - TABLESPACE pg_default; \ No newline at end of file diff --git a/src/invidious.cr b/src/invidious.cr index 41956e42..4e08e8cd 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -5,85 +5,53 @@ require "pg" require "xml" require "time" +PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious" +CONTEXT = OpenSSL::SSL::Context::Client.insecure + macro templated(filename) render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr" end class Video - getter last_updated : Time - getter video_id : String - getter video_info : String - getter video_html : String - getter views : String - getter likes : Int32 - getter dislikes : Int32 - getter rating : Float64 - getter description : String + module HTTPParamConverter + def self.from_rs(rs) + HTTP::Params.parse(rs.read(String)) + end + end - def initialize(last_updated, video_id, video_info, video_html, views, likes, dislikes, rating, description) - @last_updated = last_updated - @video_id = video_id - @video_info = video_info - @video_html = video_html - @views = views - @likes = likes - @dislikes = dislikes - @rating = rating - @description = description + module XMLConverter + def self.from_rs(rs) + XML.parse(rs.read(String)) + end + end + + def initialize(id, info, html, updated) + @id = id + @info = info + @html = html + @updated = updated end def to_a - return [@last_updated, @video_id, @video_info, @video_html, @views, @likes, @dislikes, @rating, @description] + return [@id, @info, @html, @updated] end DB.mapping({ - last_updated: Time, - video_id: String, - video_info: String, - video_html: String, - views: Int64, - likes: Int32, - dislikes: Int32, - rating: Float64, - description: String, + id: String, + info: { + type: HTTP::Params, + default: HTTP::Params.parse(""), + converter: Video::HTTPParamConverter, + }, + html: { + type: XML::Node, + default: XML.parse(""), + converter: Video::XMLConverter, + }, + updated: Time, }) end -def get_video(video_id, context) - client = HTTP::Client.new("www.youtube.com", 443, context) - video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body - info = HTTP::Params.parse(video_info) - video_html = client.get("/watch?v=#{video_id}").body - html = XML.parse(video_html) - views = info["view_count"].to_i64 - rating = info["avg_rating"].to_f64 - - likes = html.xpath_node(%q(//button[@title="I like this"]/span)) - if likes - likes = likes.content.delete(",").to_i - else - likes = 1 - end - - dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) - if dislikes - dislikes = dislikes.content.delete(",").to_i - else - dislikes = 1 - end - - description = html.xpath_node(%q(//p[@id="eow-description"])) - if description - description = description.to_xml - else - description = "" - end - - video_record = Video.new(Time.now, video_id, video_info, video_html, views, likes, dislikes, rating, description) - - return video_record -end - # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 @@ -97,15 +65,52 @@ def ci_lower_bound(pos, n) return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n) end +def fetch_video(id) + client = HTTP::Client.new("www.youtube.com", 443, CONTEXT) + info = client.get("/get_video_info?video_id=#{id}&el=info&ps=default&eurl=&gl=US&hl=en").body + info = HTTP::Params.parse(info) + + html = client.get("/watch?v=#{id}").body + html = XML.parse(html) + + if info["reason"]? + raise info["reason"] + end + + video = Video.new(id, info, html, Time.now) + + return video +end + +def get_video(id) + if PG_DB.query_one?("SELECT EXISTS ( SELECT true FROM videos WHERE id = $1)", id, as: Bool) + video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) + + # If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours) + if Time.now - video.updated > Time::Span.new(0, 5, 0, 0) + video = fetch_video(id) + end + else + video = fetch_video(id) + PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a) + end + + return video +end + get "/" do |env| templated "index" end -pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious" -context = OpenSSL::SSL::Context::Client.insecure - get "/watch" do |env| - video_id = env.params.query["v"] + id = env.params.query["v"] + + begin + video = get_video(id) + rescue ex + error_message = ex.message + next templated "error" + end if env.params.query["listen"]? && env.params.query["listen"] == "true" env.request.query_params.delete_all("listen") @@ -115,55 +120,60 @@ get "/watch" do |env| listen = false end - if pg.query_one?("select exists (select true from videos where video_id = $1)", video_id, as: Bool) - video_record = pg.query_one("select * from videos where video_id = $1", video_id, as: Video) - - # If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours) - if Time.now - video_record.last_updated > Time::Span.new(0, 5, 0, 0) - video_record = get_video(video_id, context) - pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4,\ - views = $5, likes = $6, dislikes = $7, rating = $8, description = $9 where video_id = $2", - video_record.to_a) - end - else - client = HTTP::Client.new("www.youtube.com", 443, context) - video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body - info = HTTP::Params.parse(video_info) - - if info["reason"]? - error_message = info["reason"] - next templated "error" - end - - video_record = get_video(video_id, context) - pg.exec("insert into videos values ($1,$2,$3,$4,$5,$6,$7,$8, $9)", video_record.to_a) - end - - # last_updated, video_id, video_info, video_html, views, likes, dislikes, rating - video_info = HTTP::Params.parse(video_record.video_info) - video_html = XML.parse(video_record.video_html) - fmt_stream = [] of HTTP::Params - video_info["url_encoded_fmt_stream_map"].split(",") do |string| + video.info["url_encoded_fmt_stream_map"].split(",") do |string| fmt_stream << HTTP::Params.parse(string) end - adaptive_fmts = [] of HTTP::Params - video_info["adaptive_fmts"].split(",") do |string| - adaptive_fmts << HTTP::Params.parse(string) - end - fmt_stream.reverse! # We want lowest quality first - related_videos = video_html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href)) - - if related_videos.empty? - related_videos = video_html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href)) + adaptive_fmts = [] of HTTP::Params + video.info["adaptive_fmts"].split(",") do |string| + adaptive_fmts << HTTP::Params.parse(string) end - likes = video_record.likes.to_f - dislikes = video_record.dislikes.to_f - views = video_record.views.to_f + related_videos = video.html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href)) + if related_videos.empty? + related_videos = video.html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href)) + end + + related_videos_list = [] of Video + related_videos.each do |related_video| + related_id = related_video.content.split("=")[1] + begin + related_videos_list << get_video(related_id) + rescue ex + p "#{related_id}: #{ex.message}" + end + end + + likes = video.html.xpath_node(%q(//button[@title="I like this"]/span)) + if likes + likes = likes.content.delete(",").to_i + else + likes = 1 + end + + dislikes = video.html.xpath_node(%q(//button[@title="I dislike this"]/span)) + if dislikes + dislikes = dislikes.content.delete(",").to_i + else + dislikes = 1 + end + + description = video.html.xpath_node(%q(//p[@id="eow-description"])) + if description + description = description.to_xml + else + description = "" + end + + views = video.info["view_count"].to_i64 + rating = video.info["avg_rating"].to_f64 + + likes = likes.to_f + dislikes = dislikes.to_f + views = views.to_f engagement = ((dislikes + likes)/views * 100) calculated_rating = (likes/(likes + dislikes) * 4 + 1) @@ -173,9 +183,9 @@ end get "/search" do |env| query = URI.escape(env.params.query["q"]) - client = HTTP::Client.new("www.youtube.com", 443, context) - results_html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body - html = XML.parse(results_html) + client = HTTP::Client.new("www.youtube.com", 443, CONTEXT) + html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body + html = XML.parse(html) videos = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-video-renderer)) channels = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-channel-renderer)) @@ -185,6 +195,16 @@ get "/search" do |env| channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href)) end + videos_list = [] of Video + videos.each do |video| + id = video.content.split("=")[1] + begin + videos_list << get_video(id) + rescue ex + p "#{id}: #{ex.message}" + end + end + templated "search" end diff --git a/src/views/search.ecr b/src/views/search.ecr index 3590bc7c..59dd7577 100644 --- a/src/views/search.ecr +++ b/src/views/search.ecr @@ -1,3 +1,3 @@ -<% videos.each do |video| %> -

<%= video.content %>

+<% videos_list.each do |video| %> +

<%= video.info["title"] %>

<% end %> \ No newline at end of file diff --git a/src/views/watch.ecr b/src/views/watch.ecr index 12799fb6..4e5e7b00 100644 --- a/src/views/watch.ecr +++ b/src/views/watch.ecr @@ -1,5 +1,5 @@ -<%= video_info["title"] %> - Invidious -