ProjectSegfault/invidious-experimenting
4
0
Fork 0
forked from midou/invidious
Code Pull Requests Activity

Adds description, video class, wilson score...

Browse Source
This commit is contained in:
Omar Roth 2017-12-30 15:28:41 -06:00
parent 5d48215a5f
commit f7f5f91316
1 changed files with 135 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
src/invidious.cr
View File

@ -5,41 +5,34 @@ require "pg"
require "xml" require "xml"
require "time" require "time"
class AdaptiveFmts # Get rid of everything except video_info, video_html, video_id
JSON.mapping(
clen: Int32,
url: String,
lmt: Int64,
index: String,
fps: Int32,
itag: Int32,
projection_type: Int32,
size: String,
init: String,
quality_label: String,
bitrate: Int32,
type: String
)
end
class URLEncodedFmtStreamMap # class AdaptiveFmts
JSON.mapping( # JSON.mapping(
url: String, # clen: Int32,
itag: Int32, # url: String,
fallback_host: String, # lmt: Int64,
quality: String, # index: String,
type: String # fps: Int32,
) # itag: Int32,
end # projection_type: Int32,
# size: String,
# init: String,
# quality_label: String,
# bitrate: Int32,
# type: String
# )
# end
class CaptionTracks # class URLEncodedFmtStreamMap
JSON.mapping( # JSON.mapping(
v: String, # url: String,
lc: String, # itag: Int32,
t: String, # fallback_host: String,
u: String # quality: String,
) # type: String
end # )
# end
class CaptionTranslationLanguages class CaptionTranslationLanguages
JSON.mapping( JSON.mapping(
@ -129,68 +122,131 @@ macro templated(filename)
render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr" render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr"
end end
pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious" class Video
context = OpenSSL::SSL::Context::Client.insecure getter last_updated : Time
getter video_id : String
getter video_info : String
getter video_html : String
getter views : String
getter likes : Int32
getter dislikes : Int32
getter rating : Float64
getter description : String
get "/" do |env| def initialize(last_updated, video_id, video_info, video_html, views, likes, dislikes, rating, description)
templated "index" @last_updated = last_updated
@video_id = video_id
@video_info = video_info
@video_html = video_html
@views = views
@likes = likes
@dislikes = dislikes
@rating = rating
@description = description
end
def to_a
return [@last_updated, @video_id, @video_info, @video_html, @views, @likes, @dislikes, @rating, @description]
end
DB.mapping({
last_updated: Time,
video_id: String,
video_info: String,
video_html: String,
views: Int64,
likes: Int32,
dislikes: Int32,
rating: Float64,
description: String,
})
end end
def get_record(context, video_id) def get_video(video_id, context)
client = HTTP::Client.new("www.youtube.com", 443, context) client = HTTP::Client.new("www.youtube.com", 443, context)
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
info = HTTP::Params.parse(video_info) info = HTTP::Params.parse(video_info)
video_html = client.get("/watch?v=#{video_id}").body video_html = client.get("/watch?v=#{video_id}").body
html = XML.parse(video_html) html = XML.parse(video_html)
views = info["view_count"] views = info["view_count"].to_i64
rating = info["avg_rating"].to_f64 rating = info["avg_rating"].to_f64
# css query [title = "I like this"] > span likes = html.xpath_node(%q(//button[@title="I like this"]/span))
like = html.xpath_node(%q(//button[@title="I like this"]/span)) if likes
if like likes = likes.content.delete(",").to_i
likes = like.content.delete(",").to_i
else else
likes = 1 likes = 1
end end
# css query [title = "I dislike this"] > span dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
dislike = html.xpath_node(%q(//button[@title="I dislike this"]/span)) if dislikes
if dislike dislikes = dislikes.content.delete(",").to_i
dislikes = dislike.content.delete(",").to_i
else else
dislikes = 1 dislikes = 1
end end
args = {Time.now, video_id, video_info, video_html, views, likes, dislikes, rating} description = html.xpath_node(%q(//p[@id="eow-description"]))
return args if description
description = description.to_xml
else
description = ""
end
video_record = Video.new(Time.now, video_id, video_info, video_html, views, likes, dislikes, rating, description)
return video_record
end end
get "/watch/:video_id" do |env| # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
video_id = env.params.url["video_id"] def ci_lower_bound(pos, n)
if n == 0
return 0
end
# last_updated, video_id, video_info, video_html, views, likes, dislikes, rating # z value here represents a confidence level of 0.95
video_record = pg.query_one?("select * from videos where video_id = $1", z = 1.96
video_id, phat = 1.0*pos/n
as: {Time, String, String, String, Int64, Int32, Int32, Float64})
# If record was last updated more than 5 minutes ago or doesn't exist, refresh return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n)
if video_record.nil? end
video_record = get_record(context, video_id)
pg.exec("insert into videos values ($1, $2, $3, $4, $5, $6, $7, $8)", get "/" do |env|
video_record.to_a) templated "index"
elsif Time.now.epoch - video_record[0].epoch > 300 end
video_record = get_record(context, video_id)
pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4, views = $5, likes = $6, dislikes = $7, rating = $8 where video_id = $2", pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
context = OpenSSL::SSL::Context::Client.insecure
get "/watch" do |env|
video_id = env.params.query["v"]
if pg.query_one?("select exists (select true from videos where video_id = $1)", video_id, as: Bool)
video_record = pg.query_one("select * from videos where video_id = $1", video_id, as: Video)
# If record was last updated more than 1 hour ago, refresh
if Time.now - video_record.last_updated > Time::Span.new(1, 0, 0, 0)
video_record = get_video(video_id, context)
pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4,\
views = $5, likes = $6, dislikes = $7, rating = $8, description = $9 where video_id = $2",
video_record.to_a) video_record.to_a)
end end
else
client = HTTP::Client.new("www.youtube.com", 443, context)
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
info = HTTP::Params.parse(video_info)
if info["reason"]?
error_message = info["reason"]
next templated "error"
end
video_record = get_video(video_id, context)
pg.exec("insert into videos values ($1,$2,$3,$4,$5,$6,$7,$8, $9)", video_record.to_a)
end
# last_updated, video_id, video_info, video_html, views, likes, dislikes, rating # last_updated, video_id, video_info, video_html, views, likes, dislikes, rating
video_info = HTTP::Params.parse(video_record[2]) video_info = HTTP::Params.parse(video_record.video_info)
video_html = XML.parse(video_record[3]) video_html = XML.parse(video_record.video_html)
views = video_record[4]
likes = video_record[5]
dislikes = video_record[6]
rating = video_record[7]
fmt_stream = [] of HTTP::Params fmt_stream = [] of HTTP::Params
video_info["url_encoded_fmt_stream_map"].split(",") do |string| video_info["url_encoded_fmt_stream_map"].split(",") do |string|
@ -199,16 +255,18 @@ get "/watch/:video_id" do |env|
fmt_stream.reverse! # We want lowest quality first fmt_stream.reverse! # We want lowest quality first
likes = likes.to_f related_videos = video_html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
dislikes = dislikes.to_f
views = views.to_f
engagement = ((dislikes + likes)/views * 100).significant(2) if related_videos.empty?
calculated_rating = likes/(likes + dislikes) * 4 + 1 related_videos = video_html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
end
likes = likes.to_s likes = video_record.likes.to_f
dislikes = dislikes.to_s dislikes = video_record.dislikes.to_f
views = views.to_s views = video_record.views.to_f
engagement = ((dislikes + likes)/views * 100)
calculated_rating = (likes/(likes + dislikes) * 4 + 1)
templated "watch" templated "watch"
end end