Merge branch 'master' of git.projectsegfau.lt:ProjectSegfault/ansible

all/playbook.yaml

@@ -24,6 +24,15 @@
       ansible.builtin.apt_repository:
         repo: deb [signed-by=/usr/share/keyrings/knot.gpg] https://deb.knot-dns.cz/knot-latest/ bookworm main
         state: present
+    - name: Get GoAccess GPG keys
+      ansible.builtin.get_url:
+        url: https://deb.goaccess.io/gnugpg.key
+        dest: /usr/share/keyrings/goaccess.asc
+        mode: '0644'
+    - name: Enable goaccess repo
+      ansible.builtin.apt_repository:
+        repo: deb [signed-by=/usr/share/keyrings/goaccess.asc arch=amd64] https://deb.goaccess.io/ bookworm main
+        state: present
     - name: Install Required Programs / APT
       ansible.builtin.apt:
         name:

cron/knot-restart.yaml

@@ -0,0 +1,12 @@
+---
+- name: Knot Restarts (EU/US)
+  hosts: eu,us
+  vars:
+    services:
+      - knot
+  tasks:
+    - name: Do thing
+      ansible.builtin.systemd_service:
+        state: restarted
+        name: knot 
+      with_items: "{{ services }}"

privfrontends/configs/librarian/config.yml

@@ -23,7 +23,7 @@ ENABLE_LIVESTREAM: true
 # Set custom SponsorBlock URL (with https://github.com/mchangrh/sb-mirror or other)
 SPONSORBLOCK_URL: 'https://sponsor.ajay.app'
 # Advanced: Custom video streaming endpoint
-VIDEO_STREAMING_URL: 'https://proxy.lbry.projectsegfau.lt/stream'
+VIDEO_STREAMING_URL: ''
 # Rewrite links to other frontends. example: https://yt.domain.tld
 FRONTEND:
   youtube: 'https://invidious.projectsegfau.lt'

privfrontends/configs/priviblur/config.toml

@@ -0,0 +1,68 @@
+# Controls deployment options
+[deployment]
+    host = "0.0.0.0"
+    port = 8000
+
+    # Amount of worker Priviblur instances to spawn. Increases speed significantly.
+    workers = 4
+
+    # # If you're running Priviblur behind a remote proxy, one or more of the following must be set
+    # # can also be set via env variables by captialzing and prefixing with PRIVIBLUR_
+    # #
+    # # For more information see
+    # # https://sanic.dev/en/guide/advanced/proxy-headers.html
+    # #
+    # # Default: None
+    # #
+    # forwarded_secret =
+    # real_ip_header =
+    # proxies_count =
+
+# Controls redis cache options
+# Ignore to disable the cache
+#
+[cache]
+    url = "redis://priviblur-redis:6379"
+
+    # Number of seconds to cache poll results from active polls
+    cache_active_poll_results_for = 3600
+
+    # Number of seconds to cache poll results from expired polls
+    cache_expired_poll_results_for = 86400
+
+    # Number of seconds to cache feed (explore, search, etc) results for
+    cache_feed_for = 3600
+
+    # Number of seconds to cache blog feed (blog posts, blog search, blog tagged posts, etc) results for
+    cache_blog_feed_for = 3600
+
+    # Number of seconds to cache individual posts for
+    cache_blog_post_for = 300
+
+# Controls behaviors pertaining to the way Priviblur requests Tumblr
+[priviblur_backend]
+    # # Timeout for requests to Tumblr's API
+    main_response_timeout = 10
+
+    # # Timeout for fetching image responses from Tumblr 
+    image_response_timeout = 30
+
+
+# Controls logging behavior
+#
+# Use Python's numerical logging levels
+# https://docs.python.org/3/howto/logging.html#logging-levels
+# [logging]
+    # # Sanic (Server)'s logging level'
+    # sanic_logging_level = 30
+
+    # # Priviblur's logging level
+    # priviblur_logging_level = 30
+
+    # # Priviblur extractor's logging level
+    # priviblur_extractor_logging_level = 20
+
+
+# [misc]
+    # # Enable sanic's dev mode
+    # dev_mode = false

privfrontends/configs/shoelace/shoelace.toml

@@ -0,0 +1,51 @@
+[server]
+# Address to listen on
+listen="0.0.0.0"
+# Port to bind
+port=8080
+# Instance URL. Needed for accurate proxied media locations in API
+base_url="https://lace.projectsegfau.lt"
+
+[server.tls]
+# Enable TLS support
+enabled=false
+# Path for certificate chain, in PEM format
+cert="cert.pem"
+# Path for key file, in PEM format
+key="key.pem"
+
+[endpoint]
+# Toggle the frontend
+frontend=true
+# Toggle the API
+api=true
+
+[proxy]
+# Proxy backend. Valid options are:
+# - none: Disable the media proxy. Not recommended if frontend is enabled
+# - internal: Stores values in memory. Destroys itself after stopping Shoelace.
+# - redis: Stores values in a Redis server. Higher performance. Requires additional software
+backend="internal"
+
+[proxy.redis]
+# URI for Redis server.
+# - TCP: redis://[<username>][:<password>@]<hostname>[:port][/<db>]
+# - Unix socket: redis+unix:///<path>[?db=<db>[&pass=<password>][&user=<username>]]
+uri="redis://127.0.0.1/"
+
+[logging]
+# Sets log level, for both stdout and logfiles. Valid levels are:
+# - error: Shows errors presented during runtime
+# - warn: Plus Alerts
+# - info: Plus useful information, such as PID, requests, etc. (Recommended)
+# - debug: Plus verbose actions. Not being used much.
+# - trace: Plus low-level, extremely verbose info. Not used much.
+level = "info"
+# Whether to log the IP of an incoming connection
+log_ips = false
+# Whether to log what URLs are being assigned to each hash
+log_cdn = false
+# Store logs in a text file
+store = false
+# Where to store the logs in that case
+output = "shoelace.log"

privfrontends/playbook.yaml

@@ -16,6 +16,8 @@
       when: service.value.docker_settings
 - name: Setup Caddy
   hosts: privfrontends
+  vars_files:
+    - ./blocked-ranges.yaml
   tasks:
     - name: Copy Caddyfile
       ansible.builtin.template:

privfrontends/templates/Caddyfile.j2

@@ -1,32 +1,4 @@
 {
-log {
-        # Anonymised IPs, User-Agents, and Cookies, also removed the URI as mentionned in the privacy policy.
-        # Subject to change, if we find any missing config we haven't filtered, it will be added.
-        output file /var/log/caddy/caddy.log
-                format filter {
-                   wrap json
-                   fields {
-                   request>remote_ip ip_mask {
-                           ipv4 24
-                           ipv6 64
-                   }
-                   request>headers>X-Forwarded-For ip_mask {
-                           ipv4 24
-                           ipv6 64
-                   }
-                   request>headers>client_ip ip_mask {
-                           ipv4 24
-                           ipv6 64
-                   }
-                   request>headers>Cookie cookie {
-                           replace session REDACTED
-                           delete secret
-                   }
-                   request>headers>User-Agent delete
-                   request>uri delete
-           }
-        }
-}
 	order rate_limit before basicauth
 }
 
@@ -34,7 +6,6 @@ log {
         header {
                 -Strict-Transport-Security
                 -Referrer-Policy
-                -X-XSS-Protection
                 -Content-Security-Policy
                 # disable clients from sniffing the media type
                 X-Content-Type-Options nosniff
@@ -72,10 +43,43 @@ log {
                 # clickjacking protection
                 X-Frame-Options SAMEORIGIN
 
-                X-XSS-Protection "1; mode=block"
                 defer
         }
         
+		log {
+		        # Anonymised IPs, User-Agents, and Cookies, also removed the URI as mentionned in the privacy policy.
+		        # Subject to change, if we find any missing config we haven't filtered, it will be added.
+		        output file /var/log/caddy/caddy.log
+		                format filter {
+		                   wrap json
+		                   fields {
+		                   request>remote_ip ip_mask {
+		                           ipv4 16
+		                           ipv6 64
+		                   }
+		                   request>client_ip ip_mask {
+		                           ipv4 16
+		                           ipv6 64
+		                   }
+		                   request>headers>X-Forwarded-For ip_mask {
+		                           ipv4 16
+		                           ipv6 64
+		                   }
+		                   request>headers>Cookie cookie {
+		                           replace session REDACTED
+		                           delete secret
+		                   }
+		                   request>headers>User-Agent delete
+		                   request>headers>Onion-Location delete
+		                   request>headers>Referer delete
+		                   request>uri replace "/ABCDF"
+		           }
+		        }
+		}
+		@denied client_ip {{ blocked_ranges }}
+		respond @denied "Unfortunately, your IP is part of a range that has been involved in mass spam to our servers. If you think our action was a mistake, please email contact@projectsegfau.lt." 403
+		
+
         import acmedns
 }
 
@@ -143,9 +147,19 @@ inv.{{ server_prefix }}.projectsegfau.lt i.{{ server_prefix }}.psf.lt {
                 uri @jpgRedirect replace /maxres.jpg /maxres2.jpg
                 rewrite /vi/* ?host=i.ytimg.com
         }
-		header -X-Frame-Options
 		import def
-		import torloc inv
+		header -X-Frame-Options
+		header -Content-Security-Policy
+	@ratelimit not path /videoplayback/* /ggpht/* /vi/* /videoplayback /videojs/* /css/* /js/* /fonts/* /apple-touch-icon.png /favicon-16x16.png /feed/webhook /feed/webhook/*
+	rate_limit @ratelimit {remote.ip} 10r/s
+	rate_limit @ratelimit {remote.ip} 100r/m
+	@ratelimit_pubsub path /feed/webhook /feed/webhook/*
+	rate_limit @ratelimit_pubsub {remote.ip} 2r/s 404
+        log {
+                # This is temporarily required to monitor nitter traffic due to scrapers being more active, so we need to monitor and rate limit them at a later date.
+                        output file /var/log/caddy/ratelimiters.log
+                        format json
+                }
 }
 gothub.{{ server_prefix }}.projectsegfau.lt gothub.projectsegfau.lt gh.psf.lt gh.{{ server_prefix }}.psf.lt {
         reverse_proxy :1024
@@ -222,8 +236,6 @@ search.{{ server_prefix }}.projectsegfau.lt search.projectsegfau.lt s.psf.lt s.{
         header {
                 # Enable HTTP Strict Transport Security (HSTS) to force clients to always connect via HTTPS
                 Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
-                # Enable cross-site filter (XSS) and tell browser to block detected attacks
-                X-XSS-Protection "1; mode=block"
                 # Prevent some browsers from MIME-sniffing a response away from the declared Content-Type
                 X-Content-Type-Options "nosniff"
                 # Disable some features
@@ -273,3 +285,14 @@ pi.{{ server_prefix }}.psf.lt {
 		header -X-Frame-Options
 		import def
 }
+priviblur.{{ server_prefix }}.projectsegfau.lt priviblur.projectsegfau.lt pb.psf.lt pb.{{ server_prefix }}.psf.lt {
+	import def
+	reverse_proxy :9084
+	import torloc priviblur
+}
+
+lace.{{ server_prefix }}.projectsegfau.lt lace.projectsegfau.lt l.psf.lt l.{{ server_prefix }}.psf.lt {
+	import def
+	reverse_proxy :9029
+	import torloc lace
+}

privfrontends/templates/eu/darknet.Caddyfile

@@ -139,3 +139,9 @@ http://healthchecks.pjsfkvpxlinjamtawaksbnnaqs2fc2mtvmozrzckxh7f3kis6yea25ad.oni
 		header_up Host "healthchecks.projectsegfau.lt"
 	}
 }
+http://rss.pjsfkvpxlinjamtawaksbnnaqs2fc2mtvmozrzckxh7f3kis6yea25ad.onion {
+	import tor rss
+	reverse_proxy https://rss.projectsegfau.lt {
+		header_up Host "rss.projectsegfau.lt"
+	}
+}

privfrontends/templates/eu/misc.Caddyfile

@@ -3,7 +3,7 @@ sl.projectsegfau.lt sl.psf.lt {
 	import def
 }
 inv.bp.projectsegfau.lt, i.bp.psf.lt, invidious.projectsegfau.lt, inv.projectsegfau.lt, i.psf.lt {
-	reverse_proxy localhost:7573 {
+	reverse_proxy :7573 {
 		header_up Host "invidious.projectsegfau.lt"
 	}
 	@pipedproxy {
@@ -25,6 +25,17 @@ inv.bp.projectsegfau.lt, i.bp.psf.lt, invidious.projectsegfau.lt, inv.projectseg
 	}
 	import def
 	header -X-Frame-Options
+	header -Content-Security-Policy
+	@ratelimit not path /videoplayback/* /ggpht/* /vi/* /videoplayback /videojs/* /css/* /js/* /fonts/* /apple-touch-icon.png /favicon-16x16.png /feed/webhook /feed/webhook/*
+	rate_limit @ratelimit {remote.ip} 10r/s
+	rate_limit @ratelimit {remote.ip} 100r/m
+	@ratelimit_pubsub path /feed/webhook /feed/webhook/*
+	rate_limit @ratelimit_pubsub {remote.ip} 2r/s 404
+    log {
+            # This is temporarily required to monitor nitter traffic due to scrapers being more active, so we need to monitor and rate limit them at a later date.
+                    output file /var/log/caddy/ratelimiters.log
+                    format json
+            }
 	import torloc invbp
 }
 piped.projectsegfau.lt proxy.piped.projectsegfau.lt api.piped.projectsegfau.lt {
@@ -40,7 +51,7 @@ pi.psf.lt {
 	import def
 }
 proxy.lbry.projectsegfau.lt {
-	reverse_proxy localhost:3001
+	reverse_proxy :3001
 	import def
 }
 aryak.me {
@@ -67,7 +78,7 @@ www.midou.dev midou.dev {
 # header_up Host {http.reverse_proxy.upstream.hostport} 
 # }
 #root * /var/www/midouwebsite
-reverse_proxy http://localhost:3000
+reverse_proxy :3000
 # Apparently sveltekit built apps needs to have strict path tries.
 #try_files {path} {path}/index.html {path}.html =404
 #file_server

privfrontends/templates/in/apps.Caddyfile

@@ -273,3 +273,9 @@ timetagger.projectsegfau.lt tt.projectsegfau.lt tt.psf.lt {
 	reverse_proxy :9900
 	import def
 }
+
+rss.projectsegfau.lt freshrss.projectsegfau.lt rss.psf.lt {
+	reverse_proxy :3529
+	import def
+	import torloc rss
+}

privfrontends/vars.yaml

@@ -94,15 +94,11 @@ apps:
       docker_settings:
         services:
           - name: librarian
-            image: codeberg.org/librarian/librarian:latest
+            image: quay.io/pussthecatorg/librarian
             ports:
               - "3550:3550"
             mounts:
               - "{{configs_dir}}/librarian/config.yml:/app/config.yml"
-          - name: stream-proxy
-            image: codeberg.org/librarian/stream-proxy-ng:latest
-            ports:
-              - "3001:3001"
     redlib:
       needs_data_dir: false
       needs_configs_dir: false
@@ -125,7 +121,7 @@ apps:
       docker_settings:
         services:
           - name: nitter
-            image: codeberg.org/aryak/nitter-image:latest
+            image: ghcr.io/privacydevel/nitter:master
             ports:
               - "8387:8080"
             mounts:
@@ -242,6 +238,38 @@ apps:
               REDIS_REPLICATION_MODE: master
             mounts:
               - "{{data_dir}}/teddit/redis-data:/data"
+    priviblur:
+      needs_configs_dir: true
+      needs_data_dir: true
+      docker_settings:
+        services:
+          - name: priviblur
+            image: quay.io/pussthecatorg/priviblur:latest
+            ports:
+              - "9084:8000"
+            mounts:
+              - "{{configs_dir}}/priviblur/config.toml:/priviblur/config.toml:Z,ro"
+          - name: priviblur-redis
+            image: redis:6.2.5-alpine
+            command: redis-server
+            environment:
+              REDIS_REPLICATION_MODE: master
+            mounts:
+              - "{{data_dir}}/priviblur/redis-data:/data"
+    shoelace:
+      needs_configs_dir: true
+      needs_data_dir: true
+      docker_settings:
+        services:
+          - name: shoelace
+            image: nixgoat/shoelace
+            ports:
+              - "9029:8080"
+            mounts:
+              - "{{configs_dir}}/shoelace/shoelace.toml:/data/shoelace.toml"
+              - "{{data_dir}}/shoelace:/data"
+            environment:
+              SHOELACE_CONFIG: /data/shoelace.toml
     watchtower:
       needs_configs_dir: false
       needs_data_dir: false