Manually create a sitemap.xml for docs SEO (#4261)

---------

Signed-off-by: Dmitry Sorokin <[email protected]>

docs/source/conf.py

@@ -220,9 +220,11 @@
 
 # html_extra_path used to define a path to robots.txt which is used by webcrawlers
 # to ignore or allow certain links.
-# We've decided to rely on RTD's default robots.txt, so we no longer need to specify a custom one,
-# see discussion at https://github.com/kedro-org/kedro/issues/3741
-# html_extra_path = [str(here / "robots.txt")]
+# We've tried to rely on RTD's default `robots.txt`, but it doesn't support sub-projects (Viz and Datasets),
+# so we decided to use our own `robots.txt` again, with a link to the custom `sitemap.xml`.
+# See the discussion at https://github.com/kedro-org/kedro/issues/3741
+# and https://github.com/readthedocs/readthedocs.org/issues/6841
+html_extra_path = [str(here / "extra_files")]
 
 # Removes, from all docs, the copyright footer.
 html_show_copyright = False

docs/source/extra_files/robots.txt

@@ -0,0 +1,8 @@
+User-agent: *
+Disallow: /
+
+Allow: /en/stable/
+Allow: /projects/kedro-viz/en/stable/
+Allow: /projects/kedro-datasets/en/latest/
+
+Sitemap: https://docs.kedro.org/en/stable/sitemap.xml

docs/source/sitemap.xml → docs/source/extra_files/sitemap.xml

@@ -1,34 +1,31 @@
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">
   <url>
     <loc>https://docs.kedro.org/en/stable/</loc>
-    <lastmod>2024-09-01T18:53:11.571849+00:00</lastmod>
+    <lastmod>2024-11-26T18:53:11.571849+00:00</lastmod>
     <changefreq>monthly</changefreq>
     <priority>1.0</priority>
   </url>
   <url>
     <loc>https://docs.kedro.org/en/latest/</loc>
+    <lastmod>2024-11-26T18:53:11.571849+00:00</lastmod>
     <changefreq>daily</changefreq>
     <priority>0.5</priority>
   </url>
   <url>
     <loc>https://docs.kedro.org/projects/kedro-viz/en/stable/</loc>
-    <lastmod>2024-09-01T18:53:11.571849+00:00</lastmod>
+    <lastmod>2024-11-26T18:53:11.571849+00:00</lastmod>
     <changefreq>monthly</changefreq>
     <priority>1.0</priority>
   </url>
   <url>
     <loc>https://docs.kedro.org/projects/kedro-viz/en/latest/</loc>
+    <lastmod>2024-11-26T18:53:11.571849+00:00</lastmod>
     <changefreq>daily</changefreq>
     <priority>0.5</priority>
   </url>
-  <url>
-    <loc>https://docs.kedro.org/projects/kedro-datasets/en/stable/</loc>
-    <lastmod>2024-09-01T18:53:11.571849+00:00</lastmod>
-    <changefreq>monthly</changefreq>
-    <priority>1.0</priority>
-  </url>
   <url>
     <loc>https://docs.kedro.org/projects/kedro-datasets/en/latest/</loc>
+    <lastmod>2024-11-26T18:53:11.571849+00:00</lastmod>
     <changefreq>daily</changefreq>
     <priority>0.5</priority>
   </url>