From 5e272db8f524a1cdde9c834123a5f56ad5492df4 Mon Sep 17 00:00:00 2001 From: Perflyst Date: Wed, 23 Jan 2019 20:06:43 +0100 Subject: [PATCH 01/11] Delete setup.sh --- setup.sh | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100755 setup.sh diff --git a/setup.sh b/setup.sh deleted file mode 100755 index 7b708897..00000000 --- a/setup.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -createdb invidious -#createuser kemal -psql -c "CREATE USER kemal WITH PASSWORD 'kemal';" -psql invidious < config/sql/channels.sql -psql invidious < config/sql/videos.sql -psql invidious < config/sql/channel_videos.sql -psql invidious < config/sql/users.sql -psql invidious < config/sql/nonces.sql From 03aa11b4123622284a4cfc4407d99bd94d1e8f71 Mon Sep 17 00:00:00 2001 From: Perflyst Date: Wed, 23 Jan 2019 21:12:02 +0100 Subject: [PATCH 02/11] Rewrite installation guide --- README.md | 104 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 97c9a2f8..b88b89f6 100644 --- a/README.md +++ b/README.md @@ -57,63 +57,84 @@ $ docker volume rm invidious_postgresdata $ docker-compose build ``` -### Arch Linux: +### Linux: + +#### Install dependencies ```bash -# Install dependencies -$ sudo pacman -S shards crystal imagemagick librsvg +# Arch Linux +$ sudo pacman -S shards crystal imagemagick librsvg postgresql -# Setup PostgresSQL -$ sudo systemctl enable postgresql -$ sudo systemctl start postgresql -$ sudo -i -u postgres -$ createuser -s YOUR_USER_NAME -$ createdb YOUR_USER_NAME -$ exit - -# Setup Invidious -$ git clone https://github.com/omarroth/invidious -$ cd invidious -$ ./setup.sh -$ shards -$ crystal build src/invidious.cr --release -``` - -### On Ubuntu: - -```bash -# Install dependencies +# Ubuntu or Debian $ curl -sSL https://dist.crystal-lang.org/apt/setup.sh | sudo bash $ sudo apt update $ sudo apt install crystal libssl-dev libxml2-dev libyaml-dev libgmp-dev libreadline-dev librsvg2-dev postgresql imagemagick libsqlite3-dev +``` -# Setup PostgreSQL +#### Add invidious user and clone repository + +```bash +$ useradd -m invidious +$ sudo -i -u invidious +$ git clone https://github.com/omarroth/invidious +$ exit +``` + +#### Setup PostgresSQL + +```bash $ sudo systemctl enable postgresql $ sudo systemctl start postgresql $ sudo -i -u postgres -$ createuser -s YOUR_USER_NAME_HERE -$ createdb YOUR_USER_NAME_HERE +$ psql -c "CREATE USER kemal WITH PASSWORD 'kemal';" +$ createdb -O kemal invidious +$ psql invidious < /home/invidious/invidious/config/sql/channels.sql +$ psql invidious < /home/invidious/invidious/config/sql/videos.sql +$ psql invidious < /home/invidious/invidious/config/sql/channel_videos.sql +$ psql invidious < /home/invidious/invidious/config/sql/users.sql +$ psql invidious < /home/invidious/invidious/config/sql/nonces.sql $ exit - -# Setup Invidious -$ git clone https://github.com/omarroth/invidious -$ cd invidious -$ ./setup.sh -$ shards -$ crystal build src/invidious.cr --release ``` -### On OSX: +#### Setup Invidious + +```bash +$ sudo -i -u invidious +$ cd invidious +$ shards +$ crystal build src/invidious.cr --release +# test compiled binary +$ ./invidious # stop with ctrl c +$ exit +``` + +#### systemd service +```bash +$ sudo cp invidious.service /etc/systemd/system/invidious.service +$ sudo systemctl enable invidious.service +$ sudo systemctl start invidious.service +``` + +### OSX: ```bash # Install dependencies $ brew update $ brew install shards crystal-lang postgres imagemagick librsvg -# Setup Invidious +# Clone repository and setup postgres database $ git clone https://github.com/omarroth/invidious $ cd invidious -$ ./setup.sh +$ brew services start postgresql +$ psql -c "CREATE ROLE kemal WITH LOGIN PASSWORD 'kemal';" +$ createdb invidious -U kemal +$ psql invidious < config/sql/channels.sql +$ psql invidious < config/sql/videos.sql +$ psql invidious < config/sql/channel_videos.sql +$ psql invidious < config/sql/users.sql +$ psql invidious < config/sql/nonces.sql + +# Setup Invidious $ shards $ crystal build src/invidious.cr --release ``` @@ -121,7 +142,6 @@ $ crystal build src/invidious.cr --release ## Usage: ```bash -$ crystal build src/invidious.cr --release $ ./invidious -h Usage: invidious [arguments] -b HOST, --bind HOST Host to bind (defaults to 0.0.0.0) @@ -147,16 +167,6 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta $ ./sentry ``` -## Optional - -Create a systemd service to run Invidious in background. Edit `invidious.service` to change your installation path and log location. Than copy and enable the systemd service. - -``` -$ sudo cp invidious.service /etc/systemd/system/invidious.service -$ sudo systemctl enable invidious.service -$ sudo systemctl start invidious.service -``` - ## Extensions - [Alternate Tube Redirector](https://addons.mozilla.org/en-US/firefox/addon/alternate-tube-redirector/): Automatically open Youtube Videos on alternate sites like Invidious or Hooktube. From 93297b63b153301893cc82717881505604df4aa4 Mon Sep 17 00:00:00 2001 From: Perflyst Date: Wed, 23 Jan 2019 21:31:52 +0100 Subject: [PATCH 03/11] Add logfile to systemd service and fix path --- invidious.service | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/invidious.service b/invidious.service index 556b862f..6ef12ccf 100644 --- a/invidious.service +++ b/invidious.service @@ -7,19 +7,13 @@ After=network.target RestartSec=2s Type=simple -# set user and group User=invidious Group=invidious -# configure location -WorkingDirectory=/home/invidious -ExecStart=/home/invidious/invidious/invidious +WorkingDirectory=/home/invidious/invidious +ExecStart=/home/invidious/invidious/invidious -o invidious.log Restart=always -# default log output is syslog, to disable log enable both -#StandardOutput=null -#StandardError=null - [Install] WantedBy=multi-user.target From dd5623ffbf993acc9c4bc9269ee30c4bd3716e5a Mon Sep 17 00:00:00 2001 From: Perflyst Date: Wed, 23 Jan 2019 21:40:05 +0100 Subject: [PATCH 04/11] Update invidious usage Thanks @omarroth --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b88b89f6..fd0d05d8 100644 --- a/README.md +++ b/README.md @@ -151,13 +151,14 @@ Usage: invidious [arguments] --ssl-cert-file FILE SSL certificate file -h, --help Shows this help -t THREADS, --crawl-threads=THREADS - Number of threads for crawling (default: 1) + Number of threads for crawling YouTube (default: 0) -c THREADS, --channel-threads=THREADS Number of threads for refreshing channels (default: 1) -f THREADS, --feed-threads=THREADS Number of threads for refreshing feeds (default: 1) -v THREADS, --video-threads=THREADS - Number of threads for refreshing videos (default: 1) + Number of threads for refreshing videos (default: 0) + -o OUTPUT, --output=OUTPUT Redirect output (default: STDOUT) ``` Or for development: From 16a7fcb79b627b5c9103c29b2d4604a0fedcb98c Mon Sep 17 00:00:00 2001 From: Esmail EL BoB Date: Thu, 24 Jan 2019 20:03:19 +0200 Subject: [PATCH 05/11] Update ar.json (#314) --- locales/ar.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/ar.json b/locales/ar.json index 6284043a..55592d2c 100644 --- a/locales/ar.json +++ b/locales/ar.json @@ -275,6 +275,6 @@ "Gaming": "الألعاب", "News": "الأخبار", "Movies": "الأفلام", - "Download": "", - "Download as: ": "" + "Download as:": "تحميل كـ", + "Download": "تحميل" } From 75d8c4f5c0b52b0e1743cc5cc12b4c1cc0abaf91 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 24 Jan 2019 12:16:10 -0600 Subject: [PATCH 06/11] Use logger instead of STDOUT --- src/invidious/jobs.cr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index df02c7fb..81271cb3 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -21,7 +21,7 @@ def crawl_videos(db) id = ids[0] video = get_video(id, db) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") next ensure ids.delete(id) @@ -73,7 +73,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false) db.exec("UPDATE channels SET updated = $1, author = $2 WHERE id = $3", Time.now, channel.author, id) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") end active_channel.send(true) @@ -94,7 +94,7 @@ def refresh_videos(db) id = rs.read(String) video = get_video(id, db) rescue ex - STDOUT << id << " : " << ex.message << "\n" + logger.write("#{id} : #{ex.message}\n") next end end @@ -129,7 +129,7 @@ def refresh_feeds(db, max_threads = 1) begin db.exec("REFRESH MATERIALIZED VIEW #{view_name}") rescue ex - STDOUT << "REFRESH " << email << " : " << ex.message << "\n" + logger.write("REFRESH #{email} : #{ex.message}\n") end active_channel.send(true) From 4b60f7ddff900d60a86410d40bdd55f9883b87da Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 24 Jan 2019 12:19:02 -0600 Subject: [PATCH 07/11] Add logger to method calls --- src/invidious.cr | 8 ++++---- src/invidious/jobs.cr | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 7f77e317..1a96b50a 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -108,17 +108,17 @@ LOCALES = { crawl_threads.times do spawn do - crawl_videos(PG_DB) + crawl_videos(PG_DB, logger) end end -refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh) +refresh_channels(PG_DB, logger, channel_threads, CONFIG.full_refresh) -refresh_feeds(PG_DB, feed_threads) +refresh_feeds(PG_DB, logger, feed_threads) video_threads.times do |i| spawn do - refresh_videos(PG_DB) + refresh_videos(PG_DB, logger) end end diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index 81271cb3..f6e2d8fe 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -1,4 +1,4 @@ -def crawl_videos(db) +def crawl_videos(db, logger) ids = Deque(String).new random = Random.new @@ -46,7 +46,7 @@ def crawl_videos(db) end end -def refresh_channels(db, max_threads = 1, full_refresh = false) +def refresh_channels(db, logger, max_threads = 1, full_refresh = false) max_channel = Channel(Int32).new spawn do @@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false) max_channel.send(max_threads) end -def refresh_videos(db) +def refresh_videos(db, logger) loop do db.query("SELECT id FROM videos ORDER BY updated") do |rs| rs.each do @@ -104,7 +104,7 @@ def refresh_videos(db) end end -def refresh_feeds(db, max_threads = 1) +def refresh_feeds(db, logger, max_threads = 1) max_channel = Channel(Int32).new spawn do From c1081e3df042a6097bc2a0d64d60e4adeba21543 Mon Sep 17 00:00:00 2001 From: Perflyst Date: Thu, 24 Jan 2019 19:31:56 +0100 Subject: [PATCH 08/11] Add links to documentation --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index fd0d05d8..166e050c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Onion links: - kgg2m7yk5aybusll.onion - axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid.onion +[Alternative Invidious instances](https://github.com/omarroth/invidious/wiki/Invidious-Instances) + ## Installation ### Docker: @@ -139,6 +141,9 @@ $ shards $ crystal build src/invidious.cr --release ``` +## Update Invidious +You can find information about how to update in the wiki: [Update Invidious](https://github.com/omarroth/invidious/wiki/Update-Invidious) + ## Usage: ```bash @@ -168,6 +173,9 @@ $ curl -fsSLo- https://raw.githubusercontent.com/samueleaton/sentry/master/insta $ ./sentry ``` +## Documentation +[Documentation](https://github.com/omarroth/invidious/wiki) can be found in the wiki. + ## Extensions - [Alternate Tube Redirector](https://addons.mozilla.org/en-US/firefox/addon/alternate-tube-redirector/): Automatically open Youtube Videos on alternate sites like Invidious or Hooktube. From 952b208a016548ad917737848cf3031e3f8af2d6 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 24 Jan 2019 13:52:33 -0600 Subject: [PATCH 09/11] Add retry for /videoplaybacl --- src/invidious.cr | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 1a96b50a..ad9510c1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -3644,9 +3644,23 @@ get "/videoplayback" do |env| host = "https://r#{fvip}---#{mn}.googlevideo.com" url = "/videoplayback?#{query_params.to_s}" + headers = env.request.headers + headers.delete("Host") + headers.delete("Cookie") + headers.delete("User-Agent") + headers.delete("Referer") + region = query_params["region"]? - client = make_client(URI.parse(host), proxies, region) - response = client.head(url) + + response = HTTP::Client::Response.new(403) + loop do + begin + client = make_client(URI.parse(host), proxies, region) + response = client.head(url, headers) + break + rescue ex + end + end if response.headers["Location"]? url = URI.parse(response.headers["Location"]) @@ -3664,12 +3678,6 @@ get "/videoplayback" do |env| halt env, status_code: 403 end - headers = env.request.headers - headers.delete("Host") - headers.delete("Cookie") - headers.delete("User-Agent") - headers.delete("Referer") - client = make_client(URI.parse(host), proxies, region) client.get(url, headers) do |response| env.response.status_code = response.status_code From 4c77908bb492e6d9ee567bd4c2b2fe78e63aae33 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 24 Jan 2019 19:02:09 -0600 Subject: [PATCH 10/11] Update postgres entrypoint for docker image --- docker/Dockerfile.postgres | 1 - docker/entrypoint.postgres.sh | 9 ++++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 3186e050..720bdff8 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -2,7 +2,6 @@ FROM postgres:10 ENV POSTGRES_USER postgres -ADD ./setup.sh /setup.sh ADD ./config/sql /config/sql ADD ./docker/entrypoint.postgres.sh /entrypoint.sh diff --git a/docker/entrypoint.postgres.sh b/docker/entrypoint.postgres.sh index 2f3ae65f..9a258dd6 100755 --- a/docker/entrypoint.postgres.sh +++ b/docker/entrypoint.postgres.sh @@ -10,7 +10,14 @@ if [ ! -f /var/lib/postgresql/data/setupFinished ]; then sleep 5 done >&2 echo "### importing table schemas" - su postgres -c "/setup.sh" && touch /var/lib/postgresql/data/setupFinished + su postgres -c 'createdb invidious' + su postgres -c 'psql -c "CREATE USER kemal WITH PASSWORD '"'kemal'"'"' + su postgres -c 'psql invidious < config/sql/channels.sql' + su postgres -c 'psql invidious < config/sql/videos.sql' + su postgres -c 'psql invidious < config/sql/channel_videos.sql' + su postgres -c 'psql invidious < config/sql/users.sql' + su postgres -c 'psql invidious < config/sql/nonces.sql' + touch /var/lib/postgresql/data/setupFinished echo "### invidious database setup finished" exit fi From 1af86f6afbba9a479f8d6a615c9f2a9d263d8c18 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Thu, 24 Jan 2019 20:21:19 -0600 Subject: [PATCH 11/11] Add sleep to popular_videos and top_videos --- src/invidious.cr | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/invidious.cr b/src/invidious.cr index ad9510c1..f3404d23 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -126,6 +126,8 @@ top_videos = [] of Video spawn do pull_top_videos(CONFIG, PG_DB) do |videos| top_videos = videos + sleep 1.minutes + Fiber.yield end end @@ -133,6 +135,8 @@ popular_videos = [] of ChannelVideo spawn do pull_popular_videos(PG_DB) do |videos| popular_videos = videos + sleep 1.minutes + Fiber.yield end end