Implement generic file diff (#39)

Sudoers parser has been converted to a "generic diff" parser. This parser is similar to the "generic" parser, but instead of logging changes as hashes, we'll log the actual content diff.

README.md

@@ -70,7 +70,7 @@ different consumers for four different use cases:
 it also watches for user home directory to detect ssh key injection.
 - Access consumer, just watch __/access.conf__
 - Generic consumer, watches for any existing or new files/directories for any given parent directory
-- Sudoers consumer, watches __/sudoers__ and all files under __/etc/sudoers.d__ directory.
+- Generic diff consumer, same as generic consumer, but provides diff of content instead of hashes 
 
 All consumers hold their own states to keep track of changes and diffing. If
 a difference is spotted, the diff is logged to our stdout in json format.

cfg/agent.toml

@@ -6,8 +6,9 @@ bcc = "pkg/ebpf/vfs.o"
 [consumers]
 root = "/"
 access = "/access.conf"
+genericDiff = ["/sudoers", "/etc/sudoers.d"]
 generic = ["/etc"]
-sudoers = ["/sudoers", "/etc/sudoers.d"]
+excludes = ["/etc/bookings/pool_roster"]
 
 [consumers.users]
 root = "/"

cmd/main.go

@@ -52,17 +52,17 @@ type (
 			HostRoleToken      string
 		}
 		Consumers struct {
-			Root    string
-			Access  string
-			Sudoers []string
-			Users   struct {
+			Root        string
+			Access      string
+			GenericDiff []string
+			Users       struct {
 				Shadow, Passwd string
 			}
 			Generic  []string
 			Excludes []string
 		}
 	}
-	// filesToMonitor is the struct for watching files, used for generic and sudoers consumers
+	// filesToMonitor is the struct for watching files, used for generic and generic diff consumers
 	FileInfo struct {
 		File  string
 		IsDir bool
@@ -148,18 +148,18 @@ func (c Configuration) consumers(db *pkg.AgentDB) (consumers pkg.BaseConsumers)
 			existingConsumersFiles[c.Consumers.Users.Passwd] = true
 		}
 	}
-	if len(c.Consumers.Sudoers) > 0 {
+	if len(c.Consumers.GenericDiff) > 0 {
 		//get list of files to watch
-		sudoersFiles := c.getListOfFiles(fs, c.Consumers.Sudoers)
-		for _, sudoersFile := range sudoersFiles {
-			if !c.isFileToBeExcluded(sudoersFile.File, existingConsumersFiles) {
-				state := &pkg.SudoersState{
-					SudoersListener: pkg.NewSudoersListener(
-						pkg.SudoersFileOpt(fs, sudoersFile.File, c.logger()),
+		genericDiffFiles := c.getListOfFiles(fs, c.Consumers.GenericDiff)
+		for _, genericDiffFile := range genericDiffFiles {
+			if !c.isFileToBeExcluded(genericDiffFile.File, existingConsumersFiles) {
+				state := &pkg.GenericDiffState{
+					GenericDiffListener: pkg.NewGenericDiffListener(
+						pkg.GenericDiffFileOpt(fs, genericDiffFile.File, c.logger()),
 					),
 				}
 				consumers = append(consumers, &pkg.BaseConsumer{AgentDB: db, ParserLoader: state})
-				existingConsumersFiles[sudoersFile.File] = true
+				existingConsumersFiles[genericDiffFile.File] = true
 			}
 		}
 	}
@@ -373,7 +373,7 @@ func (c Configuration) watcher() (*pkg.Watcher, error) {
 		}
 	}
 	return pkg.NewWatcher(func(w *pkg.Watcher) {
-		w.Logger, w.Consumers, w.FIM, w.Database, w.Key, w.Excludes, w.Sudoers = logger, consumers.Consumers(), fim, database, c.key, c.Consumers.Excludes, c.Consumers.Sudoers
+		w.Logger, w.Consumers, w.FIM, w.Database, w.Key, w.Excludes, w.GenericDiff = logger, consumers.Consumers(), fim, database, c.key, c.Consumers.Excludes, c.Consumers.GenericDiff
 	}), nil
 }
 

docs/README.md

@@ -16,15 +16,15 @@ of state for dedicated structures. For the moment there's only __4 types of stru
 - users
 - access
 - generic
-- sudoers
+- genericDiff
 
 For each of those the internal structure is bit different, but the way it is logged 
 is the same. Basically, when detecting a change bpfink will logs __3 different information__:
 
 - what has been added, under the `add` JSON key
 - what has been deleted, under the `del` JSON key
 - what is the current state, which as a different key depending on the consumer: 
-`users`, `generic`, `access`, `sudoers`.
+`users`, `generic`, `access`, `genericDiff`.
 
 In order to avoid complex logging logic, if an internal part of a structure has
 changed, this structure is logged both as `add` and `del`, the difference can
@@ -103,16 +103,18 @@ In this example the file dynamicPathFile was created.
 {
 	"level": "warn",
 	"add": {
-		"Sudoers": ["root ALL = (ALL:ALL) ALL"]
+		"Content": ["options timeout:2"]
 	},
 	"del": {
-		"Sudoers": []
+		"Content": []
 	},
-	"file": "bpfink.sudoers",
-	"processName": "-bash ",
+	"file": "/etc/resolv.conf",
+	"processName": "bash",
 	"user": "root",
-	"message": "Sudoers file modified"
+	"version": "0.6.51",
+	"message": "Critical Generic file modified"
 }
 ```
 
-In the above example the file bpfink.sudoers was modified.
+In the above example the file /etc/resolv.conf was modified by adding an option. Instead of the hash as seen in generic consumer,
+the diff of the content is logged.

e2etests/basic_test.go

@@ -12,7 +12,7 @@ func TestBPfink(t *testing.T) {
 	world := SetUp(t)
 	defer world.TearDown()
 	t.Run("generic file create/modify/delete", world.SubTest(testCreateGenericFile))
-	t.Run("sudoers file create", world.SubTest(testCreateSudoersDir))
+	t.Run("generic file with diff create/modify/delete", world.SubTest(testCreateGenericDiffFile))
 	t.Run("generic file in newly created dir", world.SubTest(testCreateDirectory))
 
 }
@@ -37,23 +37,23 @@ func testCreateGenericFile(t *testing.T, w *World) {
 	})
 }
 
-func testCreateSudoersDir(t *testing.T, w *World) {
-	sudoersFile := path.Join(w.FS.SudoersDir, "testSudoers")
-	f := w.FS.MustCreateFile(t, sudoersFile)
+func testCreateGenericDiffFile(t *testing.T, w *World) {
+	genericDiffFile := path.Join(w.FS.GenericMonitoringDir, "test-generic", "test-generic-diff.txt")
+	f := w.FS.MustCreateFile(t, genericDiffFile)
 	w.BPFink.ExpectEvent(t, Event{
-		File:    sudoersFile,
-		Message: "Sudoers file created",
+		File:    genericDiffFile,
+		Message: "Critical Generic file created",
 	})
 	f.WriteString("root ALL=(ALL) ALL")
 	w.BPFink.ExpectEvent(t, Event{
-		File:    sudoersFile,
-		Message: "Sudoers file modified",
+		File:    genericDiffFile,
+		Message: "Critical Generic file modified",
 	})
 
-	w.FS.MustRemoveFile(t, sudoersFile)
+	w.FS.MustRemoveFile(t, genericDiffFile)
 	w.BPFink.ExpectEvent(t, Event{
-		File:    sudoersFile,
-		Message: "Sudoers file deleted",
+		File:    genericDiffFile,
+		Message: "Critical Generic file deleted",
 	})
 }
 

e2etests/bpfink.go

@@ -26,7 +26,6 @@ type BPFinkRunParameters struct {
 	BPFinkEbpfProgramm   string
 	TestRootDir          string
 	GenericMonitoringDir string
-	SudoersDir           string
 }
 
 type baseFileLogRecord struct {
@@ -45,13 +44,13 @@ type genericFileLogRecord struct { // nolint: deadcode, unused
 	}
 }
 
-type sudoersFileLogRecord struct { // nolint: deadcode, unused
+type genericDiffFileLogRecord struct { // nolint: deadcode, unused
 	baseFileLogRecord
 	Add struct {
-		Sudoers []string
+		Content []string
 	}
 	Del struct {
-		Sudoers []string
+		Content []string
 	}
 }
 
@@ -69,7 +68,7 @@ const (
 	HEALTHY
 )
 
-func generateConfig(t *testing.T, testRootDir, genericDir string, ebpfProgrammPath string, sudoersDir string) string {
+func generateConfig(t *testing.T, testRootDir, genericDir string, ebpfProgrammPath string) string {
 	tmplt := strings.TrimSpace(`
 level = "info"
 database = "{{.TestRootDir}}/bpfink.db"
@@ -79,7 +78,7 @@ bcc = "{{.EBPfProgrammPath}}"
 [consumers]
 root = "/"
 generic = ["{{.GenericMonitoringDir}}"]
-sudoers = ["{{.SudoersDir}}"]
+genericDiff = ["{{.GenericMonitoringDir}}/test-generic"]
 `)
 
 	outConfigPath := path.Join(testRootDir, "agent.toml")
@@ -93,8 +92,7 @@ sudoers = ["{{.SudoersDir}}"]
 		TestRootDir          string
 		GenericMonitoringDir string
 		EBPfProgrammPath     string
-		SudoersDir           string
-	}{testRootDir, genericDir, ebpfProgrammPath, sudoersDir})
+	}{testRootDir, genericDir, ebpfProgrammPath})
 
 	if err != nil {
 		t.Fatalf("failed to generate config file %s: %s", outConfigPath, err)
@@ -130,16 +128,16 @@ func BPFinkRun(t *testing.T, params BPFinkRunParameters) *BPFinkInstance {
 		t.Fatalf("unable to create dir for generic monitoring: %s", err)
 	}
 
-	if err = os.Mkdir(params.SudoersDir, 0666); err != nil {
-		t.Fatalf("unable to create dir for sudoers monitoring: %s", err)
+	if err = os.Mkdir(path.Join(params.GenericMonitoringDir, "test-generic"), 0666); err != nil {
+		t.Fatalf("unable to create dir for generic file diff monitoring: %s", err)
 	}
 
 	//create broken symlink
 	if err := os.Symlink(path.Join(params.GenericMonitoringDir, "nonExistingFile.txt"), path.Join(params.GenericMonitoringDir, "brokenSymlink.txt")); err != nil {
 		t.Fatalf("unable to create symlink for file: %s", err)
 	}
 
-	configPath := generateConfig(t, params.TestRootDir, params.GenericMonitoringDir, params.BPFinkEbpfProgramm, params.SudoersDir)
+	configPath := generateConfig(t, params.TestRootDir, params.GenericMonitoringDir, params.BPFinkEbpfProgramm)
 
 	instance.cmd = exec.Command( //nolint:gosec
 		params.BPFinkBinPath,

e2etests/fs.go

@@ -7,7 +7,6 @@ import (
 
 type FS struct {
 	GenericMonitoringDir string
-	SudoersDir           string
 }
 
 func (fs *FS) MustCreateFile(t *testing.T, filePath string) *os.File {

e2etests/world.go

@@ -41,7 +41,6 @@ func SetUp(t *testing.T) *World {
 		BPFinkEbpfProgramm:   *executionParams.ebpfObjPath,
 		TestRootDir:          filepath.Join(testRootDir),
 		GenericMonitoringDir: filepath.Join(testRootDir, "root"),
-		SudoersDir:           filepath.Join(testRootDir, "sudoers"),
 	}
 
 	bpfink := BPFinkRun(t, bpfinkParams)
@@ -57,7 +56,6 @@ func SetUp(t *testing.T) *World {
 		BPFink: bpfink,
 		FS: &FS{
 			GenericMonitoringDir: bpfinkParams.GenericMonitoringDir,
-			SudoersDir:           bpfinkParams.SudoersDir,
 		},
 	}
 }

examples/watcher/setup.sh

@@ -44,12 +44,6 @@ _shadow() {
 	EOF
 }
 
-_sudoers () {
-	cat > bpfink.sudoers <<- EOF
-		root ALL = (ALL:ALL) ALL
-	EOF
-}
-
 _config () {
 	echo "bcc = \"${PROJECT}/pkg/ebpf/vfs.o\"" >> bpfink.toml
 	echo "keyfile = \"\"" >> bpfink.toml
@@ -61,8 +55,8 @@ _config () {
   echo "root = \"/\"" >>bpfink.toml
 
   echo "access = \"${PROJECT}/examples/watcher/test-dir/bpfink.access\"" >> bpfink.toml
+  echo "genericDiff = [\"${PROJECT}/examples/watcher/test-dir/dynamic-watcher/test-generic-diff-file\", \"/etc/resolv.conf\"]" >> bpfink.toml
   echo "generic = [\"${PROJECT}/examples/watcher/test-dir/dynamic-watcher\",  \"/etc\"]" >> bpfink.toml
-  echo "sudoers = [\"${PROJECT}/examples/watcher/test-dir/bpfink.sudoers\", \"/etc/sudoers.d\"]" >> bpfink.toml
   echo "excludes = [\"${PROJECT}/examples/watcher/test-dir/dynamic-watcher/exclude-dir\", \"${PROJECT}/examples/watcher/test-dir/excludeFile\"]" >>bpfink.toml
 
   cat >>bpfink.toml <<-'EOF'
@@ -90,6 +84,7 @@ init() {
   mkdir -p dynamic-watcher/dir-test
   touch dynamic-watcher/testFile
   touch dynamic-watcher/dir-test/example.txt
+  touch dynamic-watcher/test-generic-diff-file
   cd dynamic-watcher || exit
   ln -s dir-test/example.txt linked_text
   mkdir exclude-dir
@@ -99,7 +94,6 @@ init() {
   _passwd
   _shadow
   _access
-  _sudoers
   _config
   make -r -C "${PROJECT}/pkg/ebpf" || exit
 }
@@ -132,13 +126,13 @@ run_test() {
   sed -i '$ d' bpfink.shadow
   sleep 2
 
-  ##Sudoers
-  printf "\n\nadding 'ncircle ALL=(ALL) NOPASSWD:NCIRCLE_CMDS' and 'Cmnd_Alias NCIRCLE_CMDS=ALL, !/bin/netstat' to bpfink.sudoers\n\n"
-  echo "ncircle ALL=(ALL) NOPASSWD:NCIRCLE_CMDS" >>bpfink.sudoers
-  echo "Cmnd_Alias NCIRCLE_CMDS=ALL, !/bin/netstat" >>bpfink.sudoers
+  ##Generic File Diff
+  printf "\n\ntesting generic file diff\n\n"
+  echo "Test file" >>dynamic-watcher/test-generic-diff-file
+  echo "Generic file" >>dynamic-watcher/test-generic-diff-file
   sleep 2
   printf "\n\nremove last addition\n"
-  sed -i '$ d' bpfink.sudoers
+  sed -i '$ d' dynamic-watcher/test-generic-diff-file
   sleep 2
 
   printf "\n\ncreate a new file in dynamic-watcher\n\n"