From cc1c971022bb0f21ad201189143cec3a34e27c26 Mon Sep 17 00:00:00 2001
From: Perflyst <mail@perflyst.de>
Date: Tue, 1 Feb 2022 20:54:15 +0100
Subject: [PATCH] Remove robots.txt for sites we want to be indexed

Removes robots.txt for the following websites:

invidious.io - main page about the project
instances.invidious.io - instance list for end users
docs.invidious.io - docs for end users and admins
---
 templates/tin-Caddyfile.j2 | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/templates/tin-Caddyfile.j2 b/templates/tin-Caddyfile.j2
index 4a87e57..db810f2 100644
--- a/templates/tin-Caddyfile.j2
+++ b/templates/tin-Caddyfile.j2
@@ -2,11 +2,6 @@
 
 (common) {
   encode gzip
-  respond /robots.txt 200 {
-    body "User-agent: *
-Disallow: /
-"
-}
   log {
     output file /var/log/caddy/access.log {
       roll_size 500mb
@@ -25,8 +20,17 @@ Disallow: /
   }
 }
 
+(block-robots) {
+  respond /robots.txt 200 {
+    body "User-agent: *
+Disallow: /
+"
+}
+}
+
 www.invidio.us {
   import common
+  import block-robots
   redir https://invidious.io{uri}
 }
 invidious.io {
@@ -36,36 +40,43 @@ invidious.io {
 
 git.invidious.io {
   import common
+  import block-robots
   redir https://github.com/iv-org/invidious
 }
 
 invidio.us {
   import common
+  import block-robots
   redir https://redirect.invidious.io{uri}
   header /api* content-type "application/json"
   respond /api* "{\"error\":\"This server no longer hosts the Invidious API.\"}" 410
 }
 redirect.invidious.io {
   import common
+  import block-robots
   reverse_proxy http://127.0.0.1:8080
 }
 
 instances.invidio.us {
   import common
+  import block-robots
   redir https://api.invidious.io{uri}
 }
 api.invidious.io {
   import common
+  import block-robots
   reverse_proxy http://127.0.0.1:3000
   header /static* Cache-Control "max-age=86400"
 }
 
 uptime.invidio.us {
   import common
+  import block-robots
   redir https://stats.uptimerobot.com/89VnzSKAn{uri}
 }
 uptime.invidious.io {
   import common
+  import block-robots
   redir https://stats.uptimerobot.com/89VnzSKAn{uri}
 }
 
@@ -81,5 +92,6 @@ instances.invidious.io {
 
 todo.invidious.io {
   import common
+  import block-robots
   redir https://github.com/orgs/iv-org/projects
 }