feat: add extra-pages to scrape

LOQ9 · Nov 27, 2023 · 7eae728 · 7eae728
1 parent 8bc41f7
commit 7eae728
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 8 deletions.
diff --git a/cmd/wp-go-static/commands/scrape.go b/cmd/wp-go-static/commands/scrape.go
@@ -51,6 +51,7 @@ func init() {
 	ScrapeCmd.PersistentFlags().String("url", "", "URL to scrape")
 	ScrapeCmd.PersistentFlags().String("cache", "", "Cache directory")
 	ScrapeCmd.PersistentFlags().String("replace-url", "", "Replace with a specific url")
+	ScrapeCmd.PersistentFlags().StringSlice("extra-pages", []string{}, "Extra pages to scrape")
 	ScrapeCmd.PersistentFlags().Bool("replace", true, "Replace url")
 	ScrapeCmd.PersistentFlags().Bool("parallel", false, "Fetch in parallel")
 	ScrapeCmd.PersistentFlags().Bool("images", true, "Download images")
@@ -96,6 +97,11 @@ func scrapeCmdF(command *cobra.Command, args []string) error {
 	// Visit only pages that are part of the website
 	scrape.c.AllowedDomains = []string{scrape.hostname}
 
+	for _, extraPage := range scrape.config.Scrape.ExtraPages {
+		log.Println("Visiting Extra Page:", extraPage)
+		scrape.visitURL(extraPage)
+	}
+
 	// On every a element which has href attribute call callback
 	scrape.c.OnHTML("a[href]", func(e *colly.HTMLElement) {
 		link := e.Attr("href")

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -14,14 +14,15 @@ type SitemapConfig struct {
 }
 
 type ScrapeConfig struct {
-	Dir        string `mapstructure:"dir"`
-	URL        string `mapstructure:"url"`
-	Cache      string `mapstructure:"cache"`
-	ReplaceURL string `mapstructure:"replace-url"`
-	Replace    bool   `mapstructure:"replace"`
-	Parallel   bool   `mapstructure:"parallel"`
-	Images     bool   `mapstructure:"images"`
-	CheckHead  bool   `mapstructure:"check-head"`
+	Dir        string   `mapstructure:"dir"`
+	URL        string   `mapstructure:"url"`
+	Cache      string   `mapstructure:"cache"`
+	ReplaceURL string   `mapstructure:"replace-url"`
+	Replace    bool     `mapstructure:"replace"`
+	Parallel   bool     `mapstructure:"parallel"`
+	Images     bool     `mapstructure:"images"`
+	CheckHead  bool     `mapstructure:"check-head"`
+	ExtraPages []string `mapstructure:"extra-pages"`
 }
 
 type RobotsConfig struct {