diff --git a/modules/nixos-wiki/default.nix b/modules/nixos-wiki/default.nix index e34c93d..c5e2e26 100644 --- a/modules/nixos-wiki/default.nix +++ b/modules/nixos-wiki/default.nix @@ -161,6 +161,7 @@ in forceSSL = lib.mkDefault true; locations."=/nixos.png".alias = ./nixos.png; locations."=/favicon.ico".alias = ./favicon.ico; + locations."=/robots.txt".alias = ./robots.txt; }; }; } diff --git a/modules/nixos-wiki/robots.txt b/modules/nixos-wiki/robots.txt new file mode 100644 index 0000000..3c6530b --- /dev/null +++ b/modules/nixos-wiki/robots.txt @@ -0,0 +1,20 @@ +User-agent: * +Disallow: /index.php?diff= +Disallow: /index.php?oldid= +Disallow: /index.php?title=Help +Disallow: /index.php?title=Image +Disallow: /index.php?title=MediaWiki +Disallow: /index.php?title=Special: +Disallow: /index.php?title=Template +Disallow: /wiki/Special: +Disallow: /api.php +Disallow: /index.php?returnto= +Disallow: /index.php?*userExpLevel= + +# Allows robots to find all the pages containing content +Allow: /wiki/Special:AllPages + +# Allow the Internet Archiver to index action=raw and thereby store the raw wikitext of pages +User-agent: ia_archiver +Allow: /*&action=raw +Allow: /*&action=edit