ProjectSegfault/invidious-experimenting
4
0
Fork 0
forked from midou/invidious
Code Pull Requests Activity

Clarify crawl option

Browse Source
This commit is contained in:
Omar Roth 2018-05-01 18:51:16 -05:00
parent ae68213dc7
commit 58410e09f7
3 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.yml
View File

@ -1,4 +1,4 @@
threads: 5 crawl_threads: 5
channel_threads: 5 channel_threads: 5
video_threads: 5 video_threads: 5
db: db:

2
src/helpers.cr
View File

@ -15,7 +15,7 @@ end
class Config class Config
YAML.mapping({ YAML.mapping({
threads: Int32, crawl_threads: Int32,
channel_threads: Int32, channel_threads: Int32,
video_threads: Int32, video_threads: Int32,
db: NamedTuple( db: NamedTuple(

8
src/invidious.cr
View File

@ -25,15 +25,15 @@ require "./helpers"
CONFIG = Config.from_yaml(File.read("config/config.yml")) CONFIG = Config.from_yaml(File.read("config/config.yml"))
threads = CONFIG.threads crawl_threads = CONFIG.crawl_threads
channel_threads = CONFIG.channel_threads channel_threads = CONFIG.channel_threads
video_threads = CONFIG.video_threads video_threads = CONFIG.video_threads
Kemal.config.extra_options do |parser| Kemal.config.extra_options do |parser|
parser.banner = "Usage: invidious [arguments]" parser.banner = "Usage: invidious [arguments]"
parser.on("-t THREADS", "--youtube-threads=THREADS", "Number of threads for crawling (default: #{threads})") do |number| parser.on("-t THREADS", "--crawl-threads=THREADS", "Number of threads for crawling (default: #{crawl_threads})") do |number|
begin begin
threads = number.to_i crawl_threads = number.to_i
rescue ex rescue ex
puts "THREADS must be integer" puts "THREADS must be integer"
exit exit
@ -73,7 +73,7 @@ YT_URL = URI.parse("https://www.youtube.com")
REDDIT_URL = URI.parse("https://api.reddit.com") REDDIT_URL = URI.parse("https://api.reddit.com")
LOGIN_URL = URI.parse("https://accounts.google.com") LOGIN_URL = URI.parse("https://accounts.google.com")
threads.times do crawl_threads.times do
spawn do spawn do
ids = Deque(String).new ids = Deque(String).new
random = Random.new random = Random.new