This commit is contained in:
syeopite 2024-08-26 21:14:18 +00:00 committed by GitHub
commit 9d18ecdc89
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 341 additions and 166 deletions

1
.gitignore vendored
View File

@ -7,3 +7,4 @@
/invidious
/sentry
/config/config.yml
/invidious-videojs-dep-install

View File

@ -1,14 +1,253 @@
require "http"
require "yaml"
require "file_utils"
require "digest/sha1"
require "option_parser"
require "colorize"
# Represents an "install_instruction" section specified per dependency in `videojs-dependencies.yml`
#
# This is used to modify the download logic for dependencies that are packaged differently.
struct InstallInstruction
include YAML::Serializable
property js_path : String? = nil
property css_path : String? = nil
property download_as : String? = nil
property no_styling : Bool = false
end
# Object representing a dependency specified within `videojs-dependencies.yml`
class ConfigDependency
include YAML::Serializable
property version : String
property shasum : String
property install_instructions : InstallInstruction? = nil
# Checks if the current dependency needs to be installed/updated
def fetch?(name : String)
path = "assets/videojs/#{name}"
# Check for missing dependency files
#
# Does the directory exist?
# Does the Javascript file exist?
# Does the CSS file exist?
if !Dir.exists?(path)
Dir.mkdir(path)
return true
elsif !(File.exists?("#{path}/#{name}.js") || File.exists?("#{path}/versions.yml"))
return true
elsif !(self.install_instructions.try &.no_styling) && !File.exists?("#{path}/#{name}.css")
return true
end
# Check if we need to update the dependency
versions = File.open("#{path}/versions.yml") do |file|
YAML.parse(file).as_h
end
if versions["version"].as_s != self.version || versions["minified"].as_bool != CONFIG.minified
# Clear directory
{"*.js", "*.css"}.each do |file_types|
Dir.glob("#{path}/#{file_types}").each do |file_path|
File.delete(file_path)
end
end
return true
end
return false
end
end
# Object representing the `videojs-dependencies.yml` file
class PlayerDependenciesConfig
include YAML::Serializable
property version : String
property registry_url : String
property cache_directory : String
property dependencies : Hash(YAML::Any, ConfigDependency)
def get_dependencies_to_fetch
return self.dependencies.select { |name, config| config.fetch?(name.as_s) }
end
end
# Runtime Dependency config for easy access to all the variables
class Config
property minified : Bool
property skip_checksum : Bool
property clear_cache : Bool
property dependency_config : PlayerDependenciesConfig
def initialize(path : String)
@minified = false
@skip_checksum = false
@clear_cache = false
@dependency_config = PlayerDependenciesConfig.from_yaml(File.read(path))
end
# Less verbose way to access @dependency_config.registry_url
def registry_url
return @dependency_config.registry_url
end
# Less verbose way to access @dependency_config.cache_directory
def cache_directory
return @dependency_config.cache_directory
end
end
# Object representing a player dependency
class Dependency
@config : ConfigDependency
def initialize(@config : ConfigDependency, @dependency : String)
@download_path = "#{CONFIG.cache_directory}/#{@dependency}"
@destination_path = "assets/videojs/#{@dependency}"
end
private def validate_checksum(io)
return if CONFIG.skip_checksum
digest = Digest::SHA1.hexdigest(io)
if digest != @config.shasum
raise IO::Error.new("Checksum for '#{@dependency}' failed. \"#{digest}\" does not match configured \"#{@config.shasum}\"")
end
end
# Requests and downloads a specific dependency from NPM
#
# Validates a cached tarball if it already exists.
private def request_dependency
downloaded_package_path = "#{@download_path}/package.tgz"
# Create a download directory for the dependency if it does not already exist
if Dir.exists?(@download_path)
# Validate checksum of existing cached tarball
# Fetches a new one when the checksum fails.
if File.exists?(downloaded_package_path)
begin
return self.validate_checksum(File.open(downloaded_package_path))
rescue IO::Error
end
end
else
Dir.mkdir(@download_path)
end
HTTP::Client.get("#{CONFIG.registry_url}/#{@dependency}/-/#{@dependency}-#{@config.version}.tgz") do |response|
data = response.body_io.gets_to_end
File.write(downloaded_package_path, data)
self.validate_checksum(data)
end
end
# Moves a VideoJS dependency file of the given extension from extracted tarball to Invidious directory
private def move_file(full_target_path, extension)
minified_target_path = sprintf(full_target_path, {"file_extension": ".min.#{extension}"})
if CONFIG.minified && File.exists?(minified_target_path)
target_path = minified_target_path
else
target_path = sprintf(full_target_path, {"file_extension": ".#{extension}"})
end
if download_as = @config.install_instructions.try &.download_as
destination_path = "#{@destination_path}/#{sprintf(download_as, {"file_extension": ".#{extension}"})}"
else
destination_path = @destination_path
end
FileUtils.cp(target_path, destination_path)
end
# Fetch path of where a VideoJS dependency is located in the extracted tarball
private def fetch_path(is_css)
if is_css
raw_target_path = @config.install_instructions.try &.css_path
else
raw_target_path = @config.install_instructions.try &.js_path
end
if raw_target_path
return "#{@download_path}/package/#{raw_target_path}"
else
return "#{@download_path}/package/dist/#{@dependency}%{file_extension}"
end
end
# Wrapper around `#move_file` to move the dependency's JS file
private def move_js_file
return self.move_file(self.fetch_path(is_css: false), "js")
end
# Wrapper around `#move_file` to move the dependency's CSS file
#
# Does nothing with the CSS file does not exist.
private def move_css_file
path = self.fetch_path(is_css: true)
if File.exists?(sprintf(path, {"file_extension": ".css"}))
return move_file(path, "css")
end
end
# Updates the dependency's versions.yml with the current fetched version and its minified status
private def update_versions_yaml
File.open("#{@destination_path}/versions.yml", "w") do |io|
YAML.build(io) do |builder|
builder.mapping do
# Versions
builder.scalar "version"
builder.scalar "#{@config.version}"
builder.scalar "minified"
builder.scalar CONFIG.minified
end
end
end
end
# Installs a VideoJS dependency into Invidious
def install
self.request_dependency
# Crystal's stdlib provides no way of extracting a tarball
`tar -vzxf '#{@download_path}/package.tgz' -C '#{@download_path}'`
raise "Extraction for #{@dependency} failed" if !$?.success?
self.move_js_file
self.move_css_file
self.update_versions_yaml
end
end
CONFIG = Config.new("videojs-dependencies.yml")
# Hacky solution to get separated arguments when called from invidious.cr
if ARGV.size == 1
parser_args = [] of String
ARGV[0].split(",") { |str| parser_args << str.strip }
else
parser_args = ARGV
end
# Taken from https://crystal-lang.org/api/1.1.1/OptionParser.html
minified = false
OptionParser.parse do |parser|
OptionParser.parse(parser_args) do |parser|
parser.banner = "Usage: Fetch VideoJS dependencies [arguments]"
parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { minified = true }
parser.on("-m", "--minified", "Use minified versions of VideoJS dependencies (performance and bandwidth benefit)") { CONFIG.minified = true }
parser.on("--skip-checksum", "Skips the checksum validation of downloaded files") { CONFIG.skip_checksum = true }
parser.on("--clear-cache", "Clears the cache and re-downloads all dependency files") { CONFIG.clear_cache = true }
parser.on("-h", "--help", "Show this help") do
puts parser
@ -22,124 +261,17 @@ OptionParser.parse do |parser|
end
end
required_dependencies = File.open("videojs-dependencies.yml") do |file|
YAML.parse(file).as_h
end
def update_versions_yaml(required_dependencies, minified, dep_name)
File.open("assets/videojs/#{dep_name}/versions.yml", "w") do |io|
YAML.build(io) do |builder|
builder.mapping do
# Versions
builder.scalar "version"
builder.scalar "#{required_dependencies[dep_name]["version"]}"
builder.scalar "minified"
builder.scalar minified
end
end
end
end
# The first step is to check which dependencies we'll need to install.
# If the version we have requested in `videojs-dependencies.yml` is the
# same as what we've installed, we shouldn't do anything. Likewise, if it's
# different or the requested dependency just isn't present, then it needs to be
# installed.
# Since we can't know when videojs-youtube-annotations is updated, we'll just always fetch
# a new copy each time.
dependencies_to_install = [] of String
required_dependencies.keys.each do |dep|
dep = dep.to_s
path = "assets/videojs/#{dep}"
# Check for missing dependencies
if !Dir.exists?(path)
Dir.mkdir(path)
dependencies_to_install << dep
else
config = File.open("#{path}/versions.yml") do |file|
YAML.parse(file).as_h
end
if config["version"].as_s != required_dependencies[dep]["version"].as_s || config["minified"].as_bool != minified
`rm -rf #{path}/*.js #{path}/*.css`
dependencies_to_install << dep
end
end
end
# Now we begin the fun part of installing the dependencies.
# But first we'll setup a temp directory to store the plugins
tmp_dir_path = "#{Dir.tempdir}/invidious-videojs-dep-install"
Dir.mkdir(tmp_dir_path) if !Dir.exists? tmp_dir_path
# Create cache directory
Dir.mkdir(CONFIG.cache_directory) if !Dir.exists? CONFIG.cache_directory
dependencies_to_install = CONFIG.dependency_config.get_dependencies_to_fetch
channel = Channel(String | Exception).new
dependencies_to_install.each do |dep|
dependencies_to_install.each do |dep_name, dependency_config|
spawn do
dep_name = dep
download_path = "#{tmp_dir_path}/#{dep}"
dest_path = "assets/videojs/#{dep}"
HTTP::Client.get("https://registry.npmjs.org/#{dep}/-/#{dep}-#{required_dependencies[dep]["version"]}.tgz") do |response|
Dir.mkdir(download_path)
data = response.body_io.gets_to_end
File.write("#{download_path}/package.tgz", data)
# https://github.com/iv-org/invidious/pull/2397#issuecomment-922375908
if `sha1sum #{download_path}/package.tgz`.split(" ")[0] != required_dependencies[dep]["shasum"]
raise Exception.new("Checksum for '#{dep}' failed")
end
end
# Unless we install an external dependency, crystal provides no way of extracting a tarball.
# Thus we'll go ahead and call a system command.
`tar -vzxf '#{download_path}/package.tgz' -C '#{download_path}'`
raise "Extraction for #{dep} failed" if !$?.success?
# Would use File.rename in the following steps but for some reason it just doesn't work here.
# Video.js itself is structured slightly differently
dep = "video" if dep == "video.js"
# This dep nests everything under an additional JS or CSS folder
if dep == "silvermine-videojs-quality-selector"
js_path = "js/"
# It also stores their quality selector as `quality-selector.css`
`mv #{download_path}/package/dist/css/quality-selector.css #{dest_path}/quality-selector.css`
else
js_path = ""
end
# Would use File.rename but for some reason it just doesn't work here.
if minified && File.exists?("#{download_path}/package/dist/#{js_path}#{dep}.min.js")
`mv #{download_path}/package/dist/#{js_path}#{dep}.min.js #{dest_path}/#{dep}.js`
else
`mv #{download_path}/package/dist/#{js_path}#{dep}.js #{dest_path}/#{dep}.js`
end
# Fetch CSS which isn't guaranteed to exist
#
# Also, video JS changes structure here once again...
dep = "video-js" if dep == "video"
# VideoJS marker uses a dot on the CSS files.
dep = "videojs.markers" if dep == "videojs-markers"
if File.exists?("#{download_path}/package/dist/#{dep}.css")
if minified && File.exists?("#{download_path}/package/dist/#{dep}.min.css")
`mv #{download_path}/package/dist/#{dep}.min.css #{dest_path}/#{dep}.css`
else
`mv #{download_path}/package/dist/#{dep}.css #{dest_path}/#{dep}.css`
end
end
# Update/create versions file for the dependency
update_versions_yaml(required_dependencies, minified, dep_name)
channel.send(dep_name)
dependency = Dependency.new(dependency_config, dep_name.as_s)
dependency.install
channel.send(dep_name.as_s)
rescue ex
channel.send(ex)
end
@ -161,4 +293,6 @@ else
end
# Cleanup
`rm -rf #{tmp_dir_path}`
if CONFIG.clear_cache
FileUtils.rm_r("#{CONFIG.cache_directory}")
end

View File

@ -144,12 +144,26 @@ Invidious::Database.check_integrity(CONFIG)
# Running the script by itself would show some colorful feedback while this doesn't.
# Perhaps we should just move the script to runtime in order to get that feedback?
{% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %}
{% if flag?(:minified_player_dependencies) %}
{% puts run("../scripts/fetch-player-dependencies.cr", "--minified").stringify %}
{% else %}
{% puts run("../scripts/fetch-player-dependencies.cr").stringify %}
{% fetch_script_arguments = [] of StringLiteral %}
{%
potential_arguments = {
{:minified_player_dependencies, "--minified"},
{:skip_player_dependencies_checksum, "--skip-checksum"},
{:clear_player_dependencies_cache, "--clear-cache"},
}
%}
{% for potential_argument in potential_arguments %}
{% flag_, script_argument = potential_argument %}
{% if flag?(flag_) %}
{% fetch_script_arguments << script_argument.id %}
{% end %}
{% end %}
{% puts "\nChecking player dependencies, this may take more than 20 minutes... If it is stuck, check your internet connection.\n" %}
{% puts run("../scripts/fetch-player-dependencies.cr", fetch_script_arguments.splat).stringify %}
{% puts "\nDone checking player dependencies, now compiling Invidious...\n" %}
{% end %}

View File

@ -1,12 +1,12 @@
<link rel="stylesheet" href="/videojs/video.js/video-js.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/video.js/video.js.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-http-source-selector/videojs-http-source-selector.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-markers/videojs.markers.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-markers/videojs-markers.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-share/videojs-share.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-vtt-thumbnails/videojs-vtt-thumbnails.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/videojs/videojs-mobile-ui/videojs-mobile-ui.css?v=<%= ASSET_COMMIT %>">
<link rel="stylesheet" href="/css/player.css?v=<%= ASSET_COMMIT %>">
<script src="/videojs/video.js/video.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/videojs/video.js/video.js.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/videojs/videojs-mobile-ui/videojs-mobile-ui.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/videojs/videojs-contrib-quality-levels/videojs-contrib-quality-levels.js?v=<%= ASSET_COMMIT %>"></script>
<script src="/videojs/videojs-http-source-selector/videojs-http-source-selector.js?v=<%= ASSET_COMMIT %>"></script>

View File

@ -1,54 +1,80 @@
# Due to a 'video append of' error (see #3011), we're stuck on 7.12.1.
video.js:
version: 7.12.1
shasum: 1d12eeb1f52e3679e8e4c987d9b9eb37e2247fa2
version: 1
registry_url: "https://registry.npmjs.org"
videojs-contrib-quality-levels:
version: 2.1.0
shasum: 046e9e21ed01043f512b83a1916001d552457083
# Dependencies are stored as <dependency_name>/package.tgz
cache_directory: "./invidious-videojs-dep-install"
videojs-http-source-selector:
version: 1.1.6
shasum: 073aadbea0106ba6c98d6b611094dbf8554ffa1f
dependencies:
#Due to a 'video append of' error (see #3011), we're stuck on 7.12.1.
video.js:
version: 7.12.1
shasum: 1d12eeb1f52e3679e8e4c987d9b9eb37e2247fa2
videojs-markers:
version: 1.0.1
shasum: d7f8d804253fd587813271f8db308a22b9f7df34
install_instructions:
js_path: "dist/video%{file_extension}"
css_path: "dist/video-js%{file_extension}"
videojs-mobile-ui:
version: 0.6.1
shasum: 0e146c4c481cbee0729cb5e162e558b455562cd0
# Normalize names to simplify File.exists? check
download_as: "video.js%{file_extension}"
videojs-overlay:
version: 2.1.4
shasum: 5a103b25374dbb753eb87960d8360c2e8f39cc05
videojs-contrib-quality-levels:
version: 2.1.0
shasum: 046e9e21ed01043f512b83a1916001d552457083
videojs-share:
version: 3.2.1
shasum: 0a3024b981387b9d21c058c829760a72c14b8ceb
install_instructions:
no_styling: true
videojs-vr:
version: 1.8.0
shasum: 7f2f07f760d8a329c615acd316e49da6ee8edd34
videojs-http-source-selector:
version: 1.1.6
shasum: 073aadbea0106ba6c98d6b611094dbf8554ffa1f
videojs-vtt-thumbnails:
version: 0.0.13
shasum: d1e7d47f4ed80bb52f5fc4f4bad4bfc871f5970f
videojs-markers:
version: 1.0.1
shasum: d7f8d804253fd587813271f8db308a22b9f7df34
# We're using iv-org's fork of videojs-quality-selector,
# which isn't published on NPM, and doesn't have any
# easy way of fetching the compiled variant.
#
# silvermine-videojs-quality-selector:
# version: 1.1.2
# shasum: 94033ff9ee52ba6da1263b97c9a74d5b3dfdf711
install_instructions:
css_path: "dist/videojs.markers%{file_extension}"
download_as: "videojs-markers%{file_extension}"
videojs-mobile-ui:
version: 0.6.1
shasum: 0e146c4c481cbee0729cb5e162e558b455562cd0
videojs-overlay:
version: 2.1.4
shasum: 5a103b25374dbb753eb87960d8360c2e8f39cc05
videojs-share:
version: 3.2.1
shasum: 0a3024b981387b9d21c058c829760a72c14b8ceb
videojs-vr:
version: 1.8.0
shasum: 7f2f07f760d8a329c615acd316e49da6ee8edd34
videojs-vtt-thumbnails:
version: 0.0.13
shasum: d1e7d47f4ed80bb52f5fc4f4bad4bfc871f5970f
# We're using iv-org's fork of videojs-quality-selector,
# which isn't published on NPM, and doesn't have any
# easy way of fetching the compiled variant.
# silvermine-videojs-quality-selector:
# version: 1.1.2
# shasum: 94033ff9ee52ba6da1263b97c9a74d5b3dfdf711
# install_instructions:
# js_path: "dist/js/silvermine-videojs-quality-selector%{file_extension}"
# css_path: "dist/css/quality-selector%{file_extension}"
# download_as: silvermine-videojs-quality-selector%{file_extension}
# Ditto. Although this extension contains the complied variant in its git repo,
# it lacks any sort of versioning. As such, the script will ignore it.
#
# videojs-youtube-annotations:
# github: https://github.com/afrmtbl/videojs-youtube-annotations
# Ditto. Although this extension contains the complied variant in its git repo,
# it lacks any sort of versioning. As such, the script will ignore it.
#
# videojs-youtube-annotations:
# github: https://github.com/afrmtbl/videojs-youtube-annotations