fix: disallow all indexing on canary site and indexing of auth-only pages in prod (#1175)

2023-01-15 14:34:23 +00:00 · 2023-01-15 14:34:23 +00:00 · 5a6248a08f
parent 5da5f9ac7d
commit 5a6248a08f
2 changed files with 23 additions and 0 deletions
--- a/public-staging/robots.txt
+++ b/public-staging/robots.txt
@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
--- a/public/robots.txt
+++ b/public/robots.txt
@ -1,2 +1,23 @@
 User-agent: *
 Allow: /
+
+# Disallow authenticated pages
+
+Disallow /intent
+Disallow /settings
+Disallow /blocks
+Disallow /bookmarks
+Disallow /compose
+Disallow /conversations
+Disallow /domain_blocks
+Disallow /favourites
+Disallow /home
+Disallow /mutes
+Disallow /notifications
+Disallow /pinned
+Disallow /search
+Disallow /settings
+Disallow /share-target
+
+# Wait 1 second between successive requests.
+Crawl-delay: 1