Merge pull request #83 from NixOS/joerg-ci

add robots.txt
NixOS · May 18, 2024 · 981901e · 981901e
2 parents 4f40ed4 + caf2e72
commit 981901e
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 0 deletions.
diff --git a/modules/nixos-wiki/default.nix b/modules/nixos-wiki/default.nix
@@ -161,6 +161,7 @@ in
       forceSSL = lib.mkDefault true;
       locations."=/nixos.png".alias = ./nixos.png;
       locations."=/favicon.ico".alias = ./favicon.ico;
+      locations."=/robots.txt".alias = ./robots.txt;
     };
   };
 }
diff --git a/modules/nixos-wiki/robots.txt b/modules/nixos-wiki/robots.txt
@@ -0,0 +1,20 @@
+User-agent: *
+Disallow: /index.php?diff=
+Disallow: /index.php?oldid=
+Disallow: /index.php?title=Help
+Disallow: /index.php?title=Image
+Disallow: /index.php?title=MediaWiki
+Disallow: /index.php?title=Special:
+Disallow: /index.php?title=Template
+Disallow: /wiki/Special:
+Disallow: /api.php
+Disallow: /index.php?returnto=
+Disallow: /index.php?*userExpLevel=
+
+# Allows robots to find all the pages containing content
+Allow: /wiki/Special:AllPages
+
+# Allow the Internet Archiver to index action=raw and thereby store the raw wikitext of pages
+User-agent: ia_archiver
+Allow: /*&action=raw
+Allow: /*&action=edit