From 42819c90c199c3dfd2b88f3267de8fcb849d17e0 Mon Sep 17 00:00:00 2001
From: Marco Santos <me@marcopsantos.com>
Date: Fri, 1 Sep 2023 23:26:07 +0100
Subject: [PATCH] refactor: separate code

---
 internal/cache/cache.go   | 24 ++++++++++++++++++++++++
 internal/config/config.go | 24 ++++++++++++++++++++++++
 2 files changed, 48 insertions(+)
 create mode 100644 internal/cache/cache.go
 create mode 100644 internal/config/config.go

diff --git a/internal/cache/cache.go b/internal/cache/cache.go
new file mode 100644
index 0000000..8af1865
--- /dev/null
+++ b/internal/cache/cache.go
@@ -0,0 +1,24 @@
+package cache
+
+import "sync"
+
+// URLCache is a struct to hold the visited URLs
+type URLCache struct {
+	mu   sync.Mutex
+	URLs map[string]bool
+}
+
+// Add adds a URL to the cache
+func (c *URLCache) Add(url string) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.URLs[url] = true
+}
+
+// Get checks if a URL is in the cache
+func (c *URLCache) Get(url string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	_, ok := c.URLs[url]
+	return ok
+}
diff --git a/internal/config/config.go b/internal/config/config.go
new file mode 100644
index 0000000..7adb0df
--- /dev/null
+++ b/internal/config/config.go
@@ -0,0 +1,24 @@
+package config
+
+type Config struct {
+	Scrape  ScrapeConfig  `mapstructure:"scrape"`
+	Sitemap SitemapConfig `mapstructure:"sitemap"`
+}
+
+type SitemapConfig struct {
+	Dir        string `mapstructure:"dir"`
+	URL        string `mapstructure:"url"`
+	ReplaceURL string `mapstructure:"replace-url"`
+	File       string `mapstructure:"file"`
+}
+
+type ScrapeConfig struct {
+	Dir        string `mapstructure:"dir"`
+	URL        string `mapstructure:"url"`
+	Cache      string `mapstructure:"cache"`
+	ReplaceURL string `mapstructure:"replace-url"`
+	Replace    bool   `mapstructure:"replace"`
+	Parallel   bool   `mapstructure:"parallel"`
+	Images     bool   `mapstructure:"images"`
+	CheckHead  bool   `mapstructure:"check-head"`
+}