Added fs scanner

Signed-off-by: Quoc Trung Hoang <[email protected]>

README.md

@@ -8,7 +8,11 @@ Excavator is a lightweight pure Golang git leak scanning tool based on [SAP's cr
 Download a binary ![here](https://github.com/ichbinfrog/excavator/releases).
 
 ```sh
-excavator scan [flags]
+# For scanning git repository (local or remote)
+excavator gitScan [flags]
+
+# Dor scanning local directory
+excavator repoScan
 ```
 
 ### Flags
@@ -42,7 +46,7 @@ import (
 )
 
 func main() {
-  c := &scan.Scanner{}
+  c := &scan.GitScanner{}
 
   // Directory in which to store the cloned repository
   directory := ...

cmd/fs_scan.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"github.com/ichbinfrog/excavator/pkg/scan"
+	"github.com/rs/zerolog/log"
+
+	"github.com/spf13/cobra"
+)
+
+var fsScanCmd = &cobra.Command{
+	Use:   "fsScan",
+	Short: "scan a directory in the filesystem",
+	Long: `Command to scan a local directory in the filesystem.
+Will loop through each file to verify for possible password,
+access tokens (JWT, aws, gcp, ...) leaks.`,
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+
+		log.Debug().
+			Str("repo", args[0]).
+			Str("rules", rules).
+			Str("format", format).
+			Int("concurrent", concurrent).
+			Msg("Scan initiated with configuration")
+
+		s := &scan.FsScanner{}
+
+		if format == "yaml" {
+			s.New(args[0], rules, &scan.YamlReport{}, true)
+		} else {
+			s.New(args[0], rules, &scan.HTMLReport{}, true)
+		}
+		s.Scan(concurrent)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(fsScanCmd)
+}

cmd/git_scan.go

@@ -0,0 +1,41 @@
+package cmd
+
+import (
+	"github.com/ichbinfrog/excavator/pkg/scan"
+	"github.com/rs/zerolog/log"
+
+	"github.com/spf13/cobra"
+)
+
+var gitScanCmd = &cobra.Command{
+	Use:   "gitScan",
+	Short: "scan a git repository",
+	Long: `Command to scan a local or remote git repository.
+Will loop through each commit to verify for possible password,
+access tokens (JWT, aws, gcp, ...) leaks.`,
+	Args: cobra.MinimumNArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+
+		log.Debug().
+			Str("path", path).
+			Str("repo", args[0]).
+			Str("rules", rules).
+			Str("format", format).
+			Int("concurrent", concurrent).
+			Msg("Scan initiated with configuration")
+
+		s := &scan.GitScanner{}
+
+		if format == "yaml" {
+			s.New(args[0], path, rules, &scan.YamlReport{}, true)
+		} else {
+			s.New(args[0], path, rules, &scan.HTMLReport{}, true)
+		}
+		s.Scan(concurrent)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(gitScanCmd)
+	gitScanCmd.PersistentFlags().StringVarP(&path, "path", "p", ".", "temporary local path to store the git repository (only applies to remote repository)")
+}

cmd/root.go

@@ -2,20 +2,45 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
+	"github.com/mitchellh/go-homedir"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
 
+var (
+	path, rules, format string
+	concurrent          int
+)
+
 var rootCmd = &cobra.Command{
 	Use:   "excavator",
 	Short: "small cli to scan a git repository for potential leaks",
 }
 
-// Execute attempts to run the command
 func Execute() {
+	switch verbosity {
+	case 0:
+		zerolog.SetGlobalLevel(zerolog.FatalLevel)
+		break
+	case 1:
+		zerolog.SetGlobalLevel(zerolog.ErrorLevel)
+		break
+	case 3:
+		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+		break
+	case 4:
+		zerolog.SetGlobalLevel(zerolog.DebugLevel)
+		break
+	case 5:
+		zerolog.SetGlobalLevel(zerolog.TraceLevel)
+		break
+	default:
+		zerolog.SetGlobalLevel(zerolog.WarnLevel)
+	}
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal().Err(err).Msg("Failed to execute command")
 	}
@@ -31,6 +56,15 @@ func init() {
 	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
 	viper.AutomaticEnv()
 
+	home, err := homedir.Dir()
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to find $HOME directory")
+	}
+	home = filepath.Join(home, ".excavator")
+
 	flags := rootCmd.PersistentFlags()
 	flags.IntVarP(&verbosity, "verbosity", "v", 3, "logging verbosity (0: Fatal, 1: Error, 2: Warning, 3: Info, 4: Debug, 5: Trace)")
+	flags.StringVarP(&rules, "rules", "r", filepath.Join(home, "rules.yaml"), "location of the rule declaration")
+	flags.StringVarP(&format, "format", "f", "html", "output format of the scan results")
+	flags.IntVarP(&concurrent, "concurrent", "c", 1, "number of concurrent executions (any number below 0 is considered as a single routine execution)")
 }

pkg/model/leak.go

@@ -2,7 +2,7 @@ package model
 
 import "time"
 
-type Leak struct {
+type GitLeak struct {
 	Commit    string    `yaml:"commit"`
 	File      string    `yaml:"file"`
 	Line      int       `yaml:"line"`
@@ -14,3 +14,12 @@ type Leak struct {
 	Rule      *Rule     `yaml:"-"`
 	Repo      *Repo     `yaml:"-"`
 }
+
+type FileLeak struct {
+	File      string  `yaml:"file"`
+	Line      int     `yaml:"line"`
+	Size      int64   `yaml:"size"`
+	Affected  string  `yaml:"affected"`
+	Certainty float32 `yaml:"certainty,omitempty"`
+	Rule      *Rule   `yaml:"-"`
+}

pkg/model/rule.go

@@ -1,9 +1,11 @@
 package model
 
 import (
+	"bufio"
 	"encoding/hex"
 	"hash/fnv"
 	"io/ioutil"
+	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -21,10 +23,12 @@ const (
 )
 
 type RuleSet struct {
-	ApiVersion string `yaml:"apiVersion"`
-	Checksum   string
-	ReadAt     time.Time
-	Rules      []Rule `yaml:"rules"`
+	ApiVersion        string `yaml:"apiVersion"`
+	Checksum          string
+	ReadAt            time.Time
+	Rules             []Rule   `yaml:"rules"`
+	BlackList         []string `yaml:"black_list"`
+	BlackListCompiled []*regexp.Regexp
 }
 
 type Rule struct {
@@ -57,21 +61,33 @@ func (r *RuleSet) ParseConfig(file string) {
 	for idx, rule := range r.Rules {
 		r.Rules[idx].Compiled = regexp.MustCompile(rule.Definition)
 	}
+
+	for _, bl := range r.BlackList {
+		r.BlackListCompiled = append(r.BlackListCompiled, regexp.MustCompile(bl))
+	}
 }
 
-func (r *RuleSet) ParsePatch(patch *object.Patch, commit *object.Commit, repo *Repo, leakChan chan Leak) {
+func (r *RuleSet) ParsePatch(patch *object.Patch, commit *object.Commit, repo *Repo, leakChan chan GitLeak) {
 	for _, filePatch := range patch.FilePatches() {
+		if filePatch.IsBinary() {
+			break
+		}
 		_, to := filePatch.Files()
 		if to == nil {
 			continue
 		}
+		for _, blacklist := range r.BlackListCompiled {
+			if blacklist.MatchString(to.Path()) {
+				break
+			}
+		}
+
 		for _, chunk := range filePatch.Chunks() {
 			if chunk.Type() == diff.Add {
 				lines := strings.Split(strings.Replace(chunk.Content(), "\r\n", "\n", -1), "\n")
 				for idx, line := range lines {
 					for _, rule := range r.Rules {
-						match := rule.Compiled.MatchString(line)
-						if match {
+						if rule.Compiled.MatchString(line) {
 							start := idx - contextSize
 							end := idx + contextSize
 							if start < 0 {
@@ -80,7 +96,7 @@ func (r *RuleSet) ParsePatch(patch *object.Patch, commit *object.Commit, repo *R
 							if end >= len(lines) {
 								end = len(lines) - 1
 							}
-							disc := Leak{
+							disc := GitLeak{
 								Line:     idx,
 								Affected: idx - start,
 								File:     to.Path(),
@@ -101,3 +117,40 @@ func (r *RuleSet) ParsePatch(patch *object.Patch, commit *object.Commit, repo *R
 		}
 	}
 }
+
+func (r *RuleSet) ParseFile(file string, leakChan chan FileLeak) {
+	fd, err := os.Open(file)
+	if err != nil {
+		log.Error().
+			Str("file", file).
+			Err(err).
+			Msg("Failed to read")
+	}
+	scanner := bufio.NewScanner(fd)
+	defer fd.Close()
+
+	lineNum := 0
+	for scanner.Scan() {
+		lineNum++
+		for _, rule := range r.Rules {
+			line := scanner.Text()
+			if rule.Compiled.MatchString(line) {
+				stat, err := fd.Stat()
+				if err != nil {
+					log.Error().
+						Str("file", file).
+						Msg("Failed to fetch stat from")
+					continue
+				}
+				leakChan <- FileLeak{
+					File:     file,
+					Line:     lineNum,
+					Affected: line,
+					Rule:     &rule,
+					Size:     stat.Size(),
+				}
+				break
+			}
+		}
+	}
+}