refactor: separate code

internal/cache/cache.go

@@ -0,0 +1,24 @@
+package cache
+
+import "sync"
+
+// URLCache is a struct to hold the visited URLs
+type URLCache struct {
+	mu   sync.Mutex
+	URLs map[string]bool
+}
+
+// Add adds a URL to the cache
+func (c *URLCache) Add(url string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.URLs[url] = true
+}
+
+// Get checks if a URL is in the cache
+func (c *URLCache) Get(url string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	_, ok := c.URLs[url]
+	return ok
+}

internal/config/config.go

@@ -0,0 +1,24 @@
+package config
+
+type Config struct {
+	Scrape  ScrapeConfig  `mapstructure:"scrape"`
+	Sitemap SitemapConfig `mapstructure:"sitemap"`
+}
+
+type SitemapConfig struct {
+	Dir        string `mapstructure:"dir"`
+	URL        string `mapstructure:"url"`
+	ReplaceURL string `mapstructure:"replace-url"`
+	File       string `mapstructure:"file"`
+}
+
+type ScrapeConfig struct {
+	Dir        string `mapstructure:"dir"`
+	URL        string `mapstructure:"url"`
+	Cache      string `mapstructure:"cache"`
+	ReplaceURL string `mapstructure:"replace-url"`
+	Replace    bool   `mapstructure:"replace"`
+	Parallel   bool   `mapstructure:"parallel"`
+	Images     bool   `mapstructure:"images"`
+	CheckHead  bool   `mapstructure:"check-head"`
+}