2024-03-26 13:35:15 -04:00
|
|
|
Sitemap: https://redstrate.com/sitemap.xml
|
2024-02-19 14:39:10 -05:00
|
|
|
|
2024-03-26 13:35:15 -04:00
|
|
|
# Block ALL crawlers, I don't want them here
|
|
|
|
|
User-agent: *
|
2024-02-19 14:39:10 -05:00
|
|
|
Disallow: /
|
|
|
|
|
|
2024-03-26 13:35:15 -04:00
|
|
|
# Allow the two search engines I kind of care about
|
|
|
|
|
User-agent: Googlebot
|
|
|
|
|
User-agent: DuckDuckBot
|
2024-02-19 14:39:10 -05:00
|
|
|
|
2024-03-26 13:35:15 -04:00
|
|
|
# And also Matrix & Mastodon
|
|
|
|
|
User-agent: Synapse
|
|
|
|
|
User-agent: Mastodon
|
2024-02-19 14:39:10 -05:00
|
|
|
|
2024-10-12 15:38:50 -04:00
|
|
|
# Allow archive.org
|
|
|
|
|
User-agent: archive.org_bot
|
|
|
|
|
|
2024-03-26 13:35:15 -04:00
|
|
|
# Non-important pages I don't want indexed or one reason or another
|
|
|
|
|
Disallow: /shrines/
|
|
|
|
|
Disallow: /layout-archive/
|
|
|
|
|
Disallow: /imprint/
|
|
|
|
|
Disallow: /guestbook/
|
|
|
|
|
Disallow: /fund/
|
