exclude paginated pages in sitemap (#2555)

This fixes #2527.

components/config/src/config/mod.rs

@@ -29,6 +29,13 @@ pub enum Mode {
     Check,
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase")]
+pub enum ExcludePaginatedPagesInSitemap {
+    None,
+    All,
+}
+
 #[derive(Clone, Debug, Deserialize)]
 #[serde(default, deny_unknown_fields)]
 pub struct Config {
@@ -102,6 +109,8 @@ pub struct Config {
     pub generate_sitemap: bool,
     /// Enables the generation of robots.txt
     pub generate_robots_txt: bool,
+    /// Whether to exclude paginated pages in sitemap; can take values "none", "all"
+    pub exclude_paginated_pages_in_sitemap: ExcludePaginatedPagesInSitemap,
 }
 
 #[derive(Serialize)]
@@ -123,6 +132,7 @@ pub struct SerializedConfig<'a> {
     search: search::SerializedSearch<'a>,
     generate_sitemap: bool,
     generate_robots_txt: bool,
+    exclude_paginated_pages_in_sitemap: ExcludePaginatedPagesInSitemap,
 }
 
 impl Config {
@@ -287,6 +297,10 @@ impl Config {
         self.mode == Mode::Check
     }
 
+    pub fn should_exclude_paginated_pages_in_sitemap(&self) -> bool {
+        self.exclude_paginated_pages_in_sitemap == ExcludePaginatedPagesInSitemap::All
+    }
+
     pub fn enable_serve_mode(&mut self) {
         self.mode = Mode::Serve;
     }
@@ -340,6 +354,7 @@ impl Config {
             search: self.search.serialize(),
             generate_sitemap: self.generate_sitemap,
             generate_robots_txt: self.generate_robots_txt,
+            exclude_paginated_pages_in_sitemap: self.exclude_paginated_pages_in_sitemap,
         }
     }
 }
@@ -405,6 +420,7 @@ impl Default for Config {
             extra: HashMap::new(),
             generate_sitemap: true,
             generate_robots_txt: true,
+            exclude_paginated_pages_in_sitemap: ExcludePaginatedPagesInSitemap::None,
         }
     }
 }
@@ -1066,4 +1082,6 @@ base_url = "example.com"
         let config = Config::parse(config).unwrap();
         assert!(config.generate_robots_txt);
     }
+
+    // TODO: add a test for excluding paginated pages
 }

components/site/src/sitemap.rs

@@ -83,10 +83,12 @@ pub fn find_entries<'a>(
         }
 
         if let Some(paginate_by) = s.paginate_by() {
-            let number_pagers = (s.pages.len() as f64 / paginate_by as f64).ceil() as isize;
-            for i in 1..=number_pagers {
-                let permalink = format!("{}{}/{}/", s.permalink, s.meta.paginate_path, i);
-                entries.insert(SitemapEntry::new(Cow::Owned(permalink), &None));
+            if !config.should_exclude_paginated_pages_in_sitemap() {
+                let number_pagers = (s.pages.len() as f64 / paginate_by as f64).ceil() as isize;
+                for i in 1..=number_pagers {
+                    let permalink = format!("{}{}/{}/", s.permalink, s.meta.paginate_path, i);
+                    entries.insert(SitemapEntry::new(Cow::Owned(permalink), &None));
+                }
             }
         }
     }
@@ -100,7 +102,7 @@ pub fn find_entries<'a>(
         for item in &taxonomy.items {
             entries.insert(SitemapEntry::new(Cow::Borrowed(&item.permalink), &None));
 
-            if taxonomy.kind.is_paginated() {
+            if taxonomy.kind.is_paginated() && !config.should_exclude_paginated_pages_in_sitemap() {
                 let number_pagers = (item.pages.len() as f64
                     / taxonomy.kind.paginate_by.unwrap() as f64)
                     .ceil() as isize;

docs/content/documentation/getting-started/configuration.md

@@ -67,6 +67,9 @@ ignored_static = []
 # When set to "true", a feed is automatically generated.
 generate_feeds = false
 
+# When set to "all", paginated pages are not a part of the sitemap, default is "none"
+exclude_paginated_pages_in_sitemap = "none"
+
 # The filenames to use for the feeds. Used as the template filenames, too.
 # Defaults to ["atom.xml"], which has a built-in template that renders an Atom 1.0 feed.
 # There is also a built-in template "rss.xml" that renders an RSS 2.0 feed.

docs/content/documentation/templates/pagination.md

@@ -52,6 +52,12 @@ A paginated taxonomy gets two variables aside from the `paginator` variable:
 
 See the [taxonomies page](@/documentation/templates/taxonomies.md) for a detailed version of the types.
 
+## SEO
+
+It is preferable to not include paginated pages in sitemap since they are non-canonical pages.
+To exclude paginated pages in sitemap, set the
+`exclude_paginated_pages_in_sitemap` as `all` in `config.toml`.
+
 ## Example
 
 Here is an example from a theme on how to use pagination on a page (`index.html` in this case):