From a980bf3df338f225af3c8cdcb217414b31a0f58e Mon Sep 17 00:00:00 2001 From: Mikhail Scherba Date: Fri, 20 Dec 2024 16:24:55 +0300 Subject: [PATCH] chrooted bash executor Signed-off-by: Mikhail Scherba Signed-off-by: Mikhail Scherba --- pkg/addon-operator/admission_http_server.go | 7 +- pkg/addon-operator/bootstrap.go | 1 + pkg/addon-operator/operator.go | 16 +- pkg/app/app.go | 6 + .../environment_manager/evironment_manager.go | 223 +++++++++++++++++ .../evironment_manager_test.go | 226 ++++++++++++++++++ pkg/module_manager/models/hooks/dependency.go | 2 + .../models/hooks/kind/batch_hook.go | 7 +- .../models/hooks/kind/shellhook.go | 10 +- pkg/module_manager/models/modules/basic.go | 78 ++++-- pkg/module_manager/models/modules/global.go | 4 +- pkg/module_manager/module_manager.go | 35 ++- .../extenders/script_enabled/script.go | 7 +- .../scheduler/extenders/static/static.go | 2 +- pkg/utils/chroot.go | 15 ++ 15 files changed, 588 insertions(+), 51 deletions(-) create mode 100644 pkg/module_manager/environment_manager/evironment_manager.go create mode 100644 pkg/module_manager/environment_manager/evironment_manager_test.go create mode 100644 pkg/utils/chroot.go diff --git a/pkg/addon-operator/admission_http_server.go b/pkg/addon-operator/admission_http_server.go index 3a6a2e79..25a51620 100644 --- a/pkg/addon-operator/admission_http_server.go +++ b/pkg/addon-operator/admission_http_server.go @@ -2,6 +2,7 @@ package addon_operator import ( "context" + "errors" "fmt" "log/slog" "net/http" @@ -57,7 +58,11 @@ func (as *AdmissionServer) start(ctx context.Context) { cert := path.Join(as.certsDir, "tls.crt") key := path.Join(as.certsDir, "tls.key") if err := srv.ListenAndServeTLS(cert, key); err != nil { - log.Fatal("admission server listen and serve tls", log.Err(err)) + if errors.Is(err, http.ErrServerClosed) { + log.Info("admission server stopped") + } else { + log.Fatal("admission server listen and serve tls", log.Err(err)) + } } }() diff --git a/pkg/addon-operator/bootstrap.go b/pkg/addon-operator/bootstrap.go index e31df581..77b8d52f 100644 --- a/pkg/addon-operator/bootstrap.go +++ b/pkg/addon-operator/bootstrap.go @@ -84,6 +84,7 @@ func (op *AddonOperator) SetupModuleManager(modulesDir string, globalHooksDir st ModulesDir: modulesDir, GlobalHooksDir: globalHooksDir, TempDir: tempDir, + ChrootDir: app.ShellChrootDir, } deps := module_manager.ModuleManagerDependencies{ KubeObjectPatcher: op.engine.ObjectPatcher, diff --git a/pkg/addon-operator/operator.go b/pkg/addon-operator/operator.go index 6ca4521d..593fc01a 100644 --- a/pkg/addon-operator/operator.go +++ b/pkg/addon-operator/operator.go @@ -832,14 +832,16 @@ func (op *AddonOperator) HandleConvergeModules(t sh_task.Task, logLabels map[str enabledModules[enabledModule] = struct{}{} } - for _, moduleName := range op.ModuleManager.GetModuleNames() { - if _, enabled := enabledModules[moduleName]; !enabled { - op.ModuleManager.SendModuleEvent(events.ModuleEvent{ - ModuleName: moduleName, - EventType: events.ModuleDisabled, - }) + go func() { + for _, moduleName := range op.ModuleManager.GetModuleNames() { + if _, enabled := enabledModules[moduleName]; !enabled { + op.ModuleManager.SendModuleEvent(events.ModuleEvent{ + ModuleName: moduleName, + EventType: events.ModuleDisabled, + }) + } } - } + }() } tasks := op.CreateConvergeModulesTasks(state, t.GetLogLabels(), string(taskEvent)) diff --git a/pkg/app/app.go b/pkg/app/app.go index 17c49e6b..a8a4295b 100644 --- a/pkg/app/app.go +++ b/pkg/app/app.go @@ -33,6 +33,7 @@ var ( GlobalHooksDir = "global-hooks" ModulesDir = "modules" + ShellChrootDir = "" UnnumberedModuleOrder = 1 @@ -166,6 +167,11 @@ func DefineStartCommandFlags(kpApp *kingpin.Application, cmd *kingpin.CmdClause) Default(CRDsFilters). StringVar(&CRDsFilters) + cmd.Flag("shell-chroot-dir", "Defines the path where shell scripts (shell hooks and enabled scripts) will be chrooted to."). + Envar("ADDON_OPERATOR_SHELL_CHROOT_DIR"). + Default(""). + StringVar(&ShellChrootDir) + shapp.DefineKubeClientFlags(cmd) shapp.DefineJqFlags(cmd) shapp.DefineLoggingFlags(cmd) diff --git a/pkg/module_manager/environment_manager/evironment_manager.go b/pkg/module_manager/environment_manager/evironment_manager.go new file mode 100644 index 00000000..63cfa9e1 --- /dev/null +++ b/pkg/module_manager/environment_manager/evironment_manager.go @@ -0,0 +1,223 @@ +package environment_manager + +import ( + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sync" + "syscall" + + "github.com/deckhouse/deckhouse/pkg/log" + + "github.com/flant/addon-operator/pkg/utils" +) + +type ( + Type string + Environment int +) + +const ( + Mount Type = "mount" + File Type = "file" + DevNull Type = "devNull" +) + +const ( + NoEnvironment Environment = iota + EnabledScriptEnvironment + ShellHookEnvironment +) + +const testsEnv = "ADDON_OPERATOR_IS_TESTS_ENVIRONMENT" + +type ObjectDescriptor struct { + Source string + Target string + Flags uintptr + Type Type + TargetEnvironment Environment +} + +type Manager struct { + objects map[string]ObjectDescriptor + chroot string + + l sync.Mutex + preparedEnvironments map[string]Environment + + logger *log.Logger +} + +func NewManager(chroot string, logger *log.Logger) *Manager { + return &Manager{ + preparedEnvironments: make(map[string]Environment), + chroot: chroot, + objects: make(map[string]ObjectDescriptor), + logger: logger, + } +} + +func (m *Manager) AddObjectsToEnvironment(objects ...ObjectDescriptor) { + for _, object := range objects { + m.objects[object.Source] = object + } +} + +func makedev(majorNumber int64, minorNumber int64) int { + return int((majorNumber << 8) | (minorNumber & 0xff) | ((minorNumber & 0xfff00) << 12)) +} + +func (m *Manager) DisassembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error { + logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{ + "operator.component": "EnvironmentManager.DisassembleEnvironmentForModule", + }) + m.l.Lock() + defer m.l.Unlock() + + currentEnvironment := m.preparedEnvironments[moduleName] + if currentEnvironment == NoEnvironment || currentEnvironment == targetEnvironment { + return nil + } + + logEntry.Debug("Disassembling environment", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName) + for _, properties := range m.objects { + if properties.TargetEnvironment > targetEnvironment && currentEnvironment == properties.TargetEnvironment { + var chrootedObjectPath string + if len(properties.Target) > 0 { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target) + } else { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source) + } + + switch properties.Type { + case File, DevNull: + if err := os.Remove(chrootedObjectPath); err != nil { + return fmt.Errorf("delete file %q: %w", chrootedObjectPath, err) + } + + case Mount: + if err := syscall.Unmount(chrootedObjectPath, 0); err != nil { + return fmt.Errorf("unmount folder %q: %w", chrootedObjectPath, err) + } + } + } + } + + if targetEnvironment == NoEnvironment { + if os.Getenv(testsEnv) != "true" { + chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath) + if err := syscall.Unmount(chrootedModuleDir, 0); err != nil { + return fmt.Errorf("unmount %q module's dir: %w", modulePath, err) + } + } + + delete(m.preparedEnvironments, moduleName) + } else { + m.preparedEnvironments[moduleName] = targetEnvironment + } + + return nil +} + +func (m *Manager) AssembleEnvironmentForModule(moduleName, modulePath string, targetEnvironment Environment) error { + logEntry := utils.EnrichLoggerWithLabels(m.logger, map[string]string{ + "operator.component": "EnvironmentManager.PrepareEnvironmentForModule", + }) + + m.l.Lock() + defer m.l.Unlock() + + currentEnvironment := m.preparedEnvironments[moduleName] + if currentEnvironment >= targetEnvironment { + return nil + } + + logEntry.Debug("Preparing environment", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName) + + if currentEnvironment == NoEnvironment { + logEntry.Debug("Preparing environment - creating the module's directory", + slog.String("module", moduleName), + slog.Any("currentEnvironment", currentEnvironment), + slog.Any("targetEnvironment", targetEnvironment)) + + chrootedModuleDir := filepath.Join(chrootedModuleEnvPath, modulePath) + if err := os.MkdirAll(chrootedModuleDir, 0o755); err != nil { + return fmt.Errorf("make %q module's dir: %w", modulePath, err) + } + + if os.Getenv(testsEnv) != "true" { + if err := syscall.Mount(modulePath, chrootedModuleDir, "", syscall.MS_BIND|syscall.MS_RDONLY, ""); err != nil { + return fmt.Errorf("mount %q module's dir: %w", modulePath, err) + } + } + } + + for _, properties := range m.objects { + if properties.TargetEnvironment != currentEnvironment && properties.TargetEnvironment <= targetEnvironment { + var chrootedObjectPath string + if len(properties.Target) > 0 { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Target) + } else { + chrootedObjectPath = filepath.Join(chrootedModuleEnvPath, properties.Source) + } + + switch properties.Type { + case File: + if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + bytesRead, err := os.ReadFile(properties.Source) + if err != nil { + return fmt.Errorf("read from file %q: %w", properties.Source, err) + } + + if err = os.WriteFile(chrootedObjectPath, bytesRead, 0o644); err != nil { + return fmt.Errorf("write to file %q: %w", chrootedObjectPath, err) + } + + case DevNull: + if err := os.MkdirAll(filepath.Dir(chrootedObjectPath), 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + if err := syscall.Mknod(chrootedObjectPath, syscall.S_IFCHR|0o666, makedev(1, 3)); err != nil { + if errors.Is(err, os.ErrExist) { + continue + } + return fmt.Errorf("create null file: %w", err) + } + + if err := os.Chmod(chrootedObjectPath, 0o666); err != nil { + return fmt.Errorf("chmod %q file: %w", chrootedObjectPath, err) + } + + case Mount: + if err := os.MkdirAll(chrootedObjectPath, 0o755); err != nil { + return fmt.Errorf("make dir %q: %w", chrootedObjectPath, err) + } + + if err := syscall.Mount(properties.Source, chrootedObjectPath, "", properties.Flags, ""); err != nil { + return fmt.Errorf("mount folder %q: %w", chrootedObjectPath, err) + } + } + } + } + + m.preparedEnvironments[moduleName] = targetEnvironment + + return nil +} diff --git a/pkg/module_manager/environment_manager/evironment_manager_test.go b/pkg/module_manager/environment_manager/evironment_manager_test.go new file mode 100644 index 00000000..6c986122 --- /dev/null +++ b/pkg/module_manager/environment_manager/evironment_manager_test.go @@ -0,0 +1,226 @@ +package environment_manager + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/gofrs/uuid/v5" + "github.com/stretchr/testify/assert" +) + +const ( + enabledScriptFile1 = "./testdata/enabledScriptFile1" + enabledScriptFile2 = "./testdata/enabledScriptFile2" + dstEnabledScriptFile2 = "/path/to/enabledScriptFile2" + shellHookFile1 = "./testdata/shellHookFile1" + shellHookFile2 = "./testdata/shellHookFile2" + dstShellHookFile2 = "/path/to/shellHookFile2" + chrootDir = "./testdata/chroot" + + moduleName = "foo-bar" + modulePath = "./testdata/original/module/foo-bar" +) + +func TestEnvironmentManager(t *testing.T) { + os.Setenv(testsEnv, "true") + + m, uuids, err := prepareTestData() + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 0) + assert.Equal(t, NoEnvironment, m.preparedEnvironments[moduleName]) + + // prepare enabled script environment + err = m.AssembleEnvironmentForModule(moduleName, modulePath, EnabledScriptEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 1) + assert.Equal(t, EnabledScriptEnvironment, m.preparedEnvironments[moduleName]) + + // check idempotency + err = m.AssembleEnvironmentForModule(moduleName, modulePath, EnabledScriptEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 1) + assert.Equal(t, EnabledScriptEnvironment, m.preparedEnvironments[moduleName]) + + chrootedModuleEnvPath := filepath.Join(m.chroot, moduleName) + + // check if the module's dir is created + info, err := os.Stat(chrootedModuleEnvPath) + assert.NoError(t, err) + assert.True(t, info.IsDir(), "the module's original directory must be created in the chrooted environment") + + // check enabled script file 1 is copied + bytesRead, err := os.ReadFile(filepath.Join(chrootedModuleEnvPath, enabledScriptFile1)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile1], string(bytesRead)) + + // check enabled script file 2 is copied + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, dstEnabledScriptFile2)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile2], string(bytesRead)) + + // check shell hook files 1 and 2 don't exist + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, shellHookFile1)) + assert.True(t, os.IsNotExist(err), "shell hook file 1 mustn't exist") + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, dstShellHookFile2)) + assert.True(t, os.IsNotExist(err), "shell hook file 2 mustn't exist") + + // promote to shell hook environment + err = m.AssembleEnvironmentForModule(moduleName, modulePath, ShellHookEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 1) + assert.Equal(t, ShellHookEnvironment, m.preparedEnvironments[moduleName]) + + // check enabled script file 1 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, enabledScriptFile1)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile1], string(bytesRead)) + + // check enabled script file 2 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, dstEnabledScriptFile2)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile2], string(bytesRead)) + + // check shell hook file 1 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, shellHookFile1)) + assert.NoError(t, err) + assert.Equal(t, uuids[shellHookFile1], string(bytesRead)) + + // check shell hook file 2 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, dstShellHookFile2)) + assert.NoError(t, err) + assert.Equal(t, uuids[shellHookFile2], string(bytesRead)) + + // demote to enabled script environment + err = m.DisassembleEnvironmentForModule(moduleName, modulePath, EnabledScriptEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 1) + assert.Equal(t, EnabledScriptEnvironment, m.preparedEnvironments[moduleName]) + + // check enabled script file 1 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, enabledScriptFile1)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile1], string(bytesRead)) + + // check enabled script file 2 is in place + bytesRead, err = os.ReadFile(filepath.Join(chrootedModuleEnvPath, dstEnabledScriptFile2)) + assert.NoError(t, err) + assert.Equal(t, uuids[enabledScriptFile2], string(bytesRead)) + + // check shell hook files 1 and 2 don't exist + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, shellHookFile1)) + assert.True(t, os.IsNotExist(err), "shell hook file 1 mustn't exist") + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, dstShellHookFile2)) + assert.True(t, os.IsNotExist(err), "shell hook file 2 mustn't exist") + + // disassemble the environment + err = m.DisassembleEnvironmentForModule(moduleName, modulePath, NoEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 0) + assert.Equal(t, NoEnvironment, m.preparedEnvironments[moduleName]) + + // check enabled script files 1 and 2 don't exist + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, enabledScriptFile1)) + assert.True(t, os.IsNotExist(err), "enabled script file 1 mustn't exist") + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, dstEnabledScriptFile2)) + assert.True(t, os.IsNotExist(err), "enabled script file 2 mustn't exist") + + // check shell hook files 1 and 2 don't exist + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, shellHookFile1)) + assert.True(t, os.IsNotExist(err), "shell hook file 1 mustn't exist") + _, err = os.Stat(filepath.Join(chrootedModuleEnvPath, dstShellHookFile2)) + assert.True(t, os.IsNotExist(err), "shell hook file 2 mustn't exist") + + // check idempotency + err = m.DisassembleEnvironmentForModule(moduleName, modulePath, NoEnvironment) + assert.NoError(t, err) + assert.Len(t, m.preparedEnvironments, 0) + assert.Equal(t, NoEnvironment, m.preparedEnvironments[moduleName]) + + err = cleanupTestData() + assert.NoError(t, err) +} + +func prepareTestData() (*Manager, map[string]string, error) { + if err := os.Mkdir("./testdata", 0o750); err != nil { + return nil, nil, fmt.Errorf("create testdata dir: %w", err) + } + + uuids := map[string]string{ + enabledScriptFile1: "", + enabledScriptFile2: "", + shellHookFile1: "", + shellHookFile2: "", + } + + for file := range uuids { + uuid, err := uuid.NewV4() + if err != nil { + return nil, nil, fmt.Errorf("generate uuid: %w", err) + } + + uuids[file] = uuid.String() + if err = os.WriteFile(file, []byte(uuid.String()), 0o644); err != nil { + return nil, nil, fmt.Errorf("write uuid to file %s: %w", file, err) + } + } + + m := NewManager(chrootDir, log.NewNop()) + m.AddObjectsToEnvironment([]ObjectDescriptor{ + { + Source: enabledScriptFile1, + Type: File, + TargetEnvironment: EnabledScriptEnvironment, + }, + { + Source: enabledScriptFile2, + Type: File, + Target: dstEnabledScriptFile2, + TargetEnvironment: EnabledScriptEnvironment, + }, + { + Source: shellHookFile1, + Type: File, + TargetEnvironment: ShellHookEnvironment, + }, + { + Source: shellHookFile2, + Type: File, + Target: dstShellHookFile2, + TargetEnvironment: ShellHookEnvironment, + }, + }...) + + return m, uuids, nil +} + +func cleanupTestData() error { + err := os.Remove(enabledScriptFile1) + if err != nil { + return fmt.Errorf("delete testdata file %s: %w", enabledScriptFile1, err) + } + + err = os.Remove(enabledScriptFile2) + if err != nil { + return fmt.Errorf("delete testdata file %s: %w", enabledScriptFile2, err) + } + + err = os.Remove(shellHookFile1) + if err != nil { + return fmt.Errorf("delete testdata file %s: %w", shellHookFile1, err) + } + + err = os.Remove(shellHookFile2) + if err != nil { + return fmt.Errorf("delete testdata file %s: %w", shellHookFile2, err) + } + + err = os.RemoveAll("./testdata") + if err != nil { + return fmt.Errorf("delete testdata dir: %w", err) + } + + return nil +} diff --git a/pkg/module_manager/models/hooks/dependency.go b/pkg/module_manager/models/hooks/dependency.go index 479b4735..5379b8d8 100644 --- a/pkg/module_manager/models/hooks/dependency.go +++ b/pkg/module_manager/models/hooks/dependency.go @@ -3,6 +3,7 @@ package hooks import ( "context" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" gohook "github.com/flant/addon-operator/pkg/module_manager/go_hook" "github.com/flant/addon-operator/pkg/module_manager/models/hooks/kind" "github.com/flant/addon-operator/pkg/utils" @@ -42,6 +43,7 @@ type HookExecutionDependencyContainer struct { KubeObjectPatcher kubeObjectPatcher MetricStorage metricStorage GlobalValuesGetter globalValuesGetter + EnvironmentManager *environmentmanager.Manager } type executableHook interface { diff --git a/pkg/module_manager/models/hooks/kind/batch_hook.go b/pkg/module_manager/models/hooks/kind/batch_hook.go index ad2d9d53..508134c6 100644 --- a/pkg/module_manager/models/hooks/kind/batch_hook.go +++ b/pkg/module_manager/models/hooks/kind/batch_hook.go @@ -30,6 +30,7 @@ import ( var _ gohook.HookConfigLoader = (*BatchHook)(nil) type BatchHook struct { + moduleName string sh_hook.Hook // hook ID in batch ID uint @@ -37,8 +38,9 @@ type BatchHook struct { } // NewBatchHook new hook, which runs via the OS interpreter like bash/python/etc -func NewBatchHook(name, path string, id uint, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *BatchHook { +func NewBatchHook(name, path, moduleName string, id uint, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *BatchHook { return &BatchHook{ + moduleName: moduleName, Hook: sh_hook.Hook{ Name: name, Path: path, @@ -143,7 +145,8 @@ func (h *BatchHook) Execute(configVersion string, bContext []bindingcontext.Bind envs). WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(h.LogProxyHookJSONKey). - WithLogger(h.Logger.Named("executor")) + WithLogger(h.Logger.Named("executor")). + WithChroot(utils.GetModuleChrootPath(h.moduleName)) usage, err := cmd.RunAndLogLines(logLabels) result.Usage = usage diff --git a/pkg/module_manager/models/hooks/kind/shellhook.go b/pkg/module_manager/models/hooks/kind/shellhook.go index 7ecc2998..5c417a94 100644 --- a/pkg/module_manager/models/hooks/kind/shellhook.go +++ b/pkg/module_manager/models/hooks/kind/shellhook.go @@ -27,14 +27,16 @@ import ( var _ gohook.HookConfigLoader = (*ShellHook)(nil) type ShellHook struct { + moduleName string sh_hook.Hook ScheduleConfig *HookScheduleConfig } // NewShellHook new hook, which runs via the OS interpreter like bash/python/etc -func NewShellHook(name, path string, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *ShellHook { +func NewShellHook(name, path, moduleName string, keepTemporaryHookFiles bool, logProxyHookJSON bool, logger *log.Logger) *ShellHook { return &ShellHook{ + moduleName: moduleName, Hook: sh_hook.Hook{ Name: name, Path: path, @@ -136,7 +138,8 @@ func (sh *ShellHook) Execute(configVersion string, bContext []bindingcontext.Bin envs). WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(sh.LogProxyHookJSONKey). - WithLogger(sh.Logger.Named("executor")) + WithLogger(sh.Logger.Named("executor")). + WithChroot(utils.GetModuleChrootPath(sh.moduleName)) usage, err := cmd.RunAndLogLines(logLabels) result.Usage = usage @@ -185,7 +188,8 @@ func (sh *ShellHook) getConfig() ([]byte, error) { WithLogProxyHookJSON(shapp.LogProxyHookJSON). WithLogProxyHookJSONKey(sh.LogProxyHookJSONKey). WithLogger(sh.Logger.Named("executor")). - WithCMDStdout(nil) + WithCMDStdout(nil). + WithChroot(utils.GetModuleChrootPath(sh.moduleName)) sh.Hook.Logger.Debug("Executing hook", slog.String("args", strings.Join(args, " "))) diff --git a/pkg/module_manager/models/modules/basic.go b/pkg/module_manager/models/modules/basic.go index e25064d6..4356cec6 100644 --- a/pkg/module_manager/models/modules/basic.go +++ b/pkg/module_manager/models/modules/basic.go @@ -21,6 +21,7 @@ import ( "github.com/flant/addon-operator/pkg/app" "github.com/flant/addon-operator/pkg/hook/types" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" "github.com/flant/addon-operator/pkg/module_manager/models/hooks" "github.com/flant/addon-operator/pkg/module_manager/models/hooks/kind" "github.com/flant/addon-operator/pkg/utils" @@ -192,18 +193,31 @@ func (bm *BasicModule) ResetState() { } } -// RegisterHooks find and registers all module hooks from a filesystem or GoHook Registry +// RegisterHooks searches and registers all module hooks from a filesystem or GoHook Registry func (bm *BasicModule) RegisterHooks(logger *log.Logger) ([]*hooks.ModuleHook, error) { if bm.hooks.registered { logger.Debug("Module hooks already registered") return nil, nil } - logger.Debug("Search and register hooks") - - hks, err := bm.searchAndRegisterHooks(logger) + hks, err := bm.searchModuleHooks() if err != nil { - return nil, fmt.Errorf("search and register hooks: %w", err) + return nil, fmt.Errorf("search module hooks failed: %w", err) + } + + logger.Debug("Found hooks", slog.Int("count", len(hks))) + if logger.GetLevel() == log.LevelDebug { + for _, h := range hks { + logger.Debug("ModuleHook", + slog.String("name", h.GetName()), + slog.String("path", h.GetPath())) + } + } + + logger.Debug("Register hooks") + + if err := bm.registerHooks(hks, logger); err != nil { + return nil, fmt.Errorf("register hooks: %w", err) } bm.hooks.registered = true @@ -224,6 +238,12 @@ func (bm *BasicModule) searchModuleHooks() ([]*hooks.ModuleHook, error) { return nil, fmt.Errorf("search module batch hooks: %w", err) } + if len(shellHooks)+len(batchHooks) > 0 { + if err := bm.AssembleEnvironmentForModule(environmentmanager.ShellHookEnvironment); err != nil { + return nil, fmt.Errorf("Assemble %q module's environment: %w", bm.Name, err) + } + } + mHooks := make([]*hooks.ModuleHook, 0, len(shellHooks)+len(goHooks)) for _, sh := range shellHooks { @@ -281,7 +301,7 @@ func (bm *BasicModule) searchModuleShellHooks() ([]*kind.ShellHook, error) { bm.logger.Warn("get batch hook config", slog.String("hook_file_path", hookPath), log.Err(err)) } - shHook := kind.NewShellHook(hookName, hookPath, bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("shell-hook")) + shHook := kind.NewShellHook(hookName, hookPath, bm.Name, bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("shell-hook")) hks = append(hks, shHook) } @@ -319,7 +339,7 @@ func (bm *BasicModule) searchModuleBatchHooks() ([]*kind.BatchHook, error) { for idx, cfg := range sdkcfgs { nestedHookName := fmt.Sprintf("%s:%s:%d", hookName, cfg.Metadata.Name, idx) - shHook := kind.NewBatchHook(nestedHookName, hookPath, uint(idx), bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("batch-hook")) + shHook := kind.NewBatchHook(nestedHookName, hookPath, bm.Name, uint(idx), bm.keepTemporaryHookFiles, shapp.LogProxyHookJSON, bm.logger.Named("batch-hook")) hks = append(hks, shHook) } @@ -393,6 +413,7 @@ func IsFileBatchHook(path string, f os.FileInfo) error { // TODO: check binary another way args := []string{"hook", "list"} + o, err := exec.Command(path, args...).Output() if err != nil { return fmt.Errorf("exec file '%s': %w", path, err) @@ -410,21 +431,7 @@ func (bm *BasicModule) searchModuleGoHooks() []*kind.GoHook { return sdk.Registry().GetModuleHooks(bm.Name) } -func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.ModuleHook, error) { - hks, err := bm.searchModuleHooks() - if err != nil { - return nil, fmt.Errorf("search module hooks failed: %w", err) - } - - logger.Debug("Found hooks", slog.Int("count", len(hks))) - if logger.GetLevel() == log.LevelDebug { - for _, h := range hks { - logger.Debug("ModuleHook", - slog.String("name", h.GetName()), - slog.String("path", h.GetPath())) - } - } - +func (bm *BasicModule) registerHooks(hks []*hooks.ModuleHook, logger *log.Logger) error { for _, moduleHook := range hks { hookLogEntry := logger.With("hook", moduleHook.GetName()). With("hook.type", "module") @@ -432,7 +439,7 @@ func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.Modu // TODO: we could make multierr here and return all config errors at once err := moduleHook.InitializeHookConfig() if err != nil { - return nil, fmt.Errorf("module hook --config invalid: %w", err) + return fmt.Errorf("module hook --config invalid: %w", err) } bm.logger.Debug("module hook config print", slog.String("module_name", bm.Name), slog.String("hook_name", moduleHook.GetName()), slog.Any("config", moduleHook.GetHookConfig().V1)) @@ -459,7 +466,7 @@ func (bm *BasicModule) searchAndRegisterHooks(logger *log.Logger) ([]*hooks.Modu slog.String("bindings", moduleHook.GetConfigDescription())) } - return hks, nil + return nil } // GetPhase ... @@ -647,13 +654,18 @@ func (bm *BasicModule) RunEnabledScript(tmpDir string, precedingEnabledModules [ envs = append(envs, fmt.Sprintf("VALUES_PATH=%s", valuesPath)) envs = append(envs, fmt.Sprintf("MODULE_ENABLED_RESULT=%s", enabledResultFilePath)) + if err := bm.AssembleEnvironmentForModule(environmentmanager.EnabledScriptEnvironment); err != nil { + return false, fmt.Errorf("Assemble %q module's environment: %w", bm.Name, err) + } + cmd := executor.NewExecutor( "", enabledScriptPath, []string{}, envs). WithLogger(bm.logger.Named("executor")). - WithCMDStdout(nil) + WithCMDStdout(nil). + WithChroot(utils.GetModuleChrootPath(bm.Name)) usage, err := cmd.RunAndLogLines(logLabels) if usage != nil { @@ -1130,6 +1142,22 @@ func (bm *BasicModule) Validate() error { return nil } +func (bm *BasicModule) DisassembleEnvironmentForModule() error { + if bm.dc.EnvironmentManager != nil { + return bm.dc.EnvironmentManager.DisassembleEnvironmentForModule(bm.Name, bm.Path, environmentmanager.NoEnvironment) + } + + return nil +} + +func (bm *BasicModule) AssembleEnvironmentForModule(targetEnvironment environmentmanager.Environment) error { + if bm.dc.EnvironmentManager != nil { + return bm.dc.EnvironmentManager.AssembleEnvironmentForModule(bm.Name, bm.Path, targetEnvironment) + } + + return nil +} + func (bm *BasicModule) ValidateValues() error { return bm.valuesStorage.validateValues(bm.GetValues(false)) } diff --git a/pkg/module_manager/models/modules/global.go b/pkg/module_manager/models/modules/global.go index 17df4f37..c988c40b 100644 --- a/pkg/module_manager/models/modules/global.go +++ b/pkg/module_manager/models/modules/global.go @@ -548,7 +548,7 @@ func (gm *GlobalModule) searchGlobalShellHooks(hooksDir string) ([]*kind.ShellHo } } - globalHook := kind.NewShellHook(hookName, hookPath, gm.keepTemporaryHookFiles, false, gm.logger.Named("shell-hook")) + globalHook := kind.NewShellHook(hookName, hookPath, "global", gm.keepTemporaryHookFiles, false, gm.logger.Named("shell-hook")) hks = append(hks, globalHook) } @@ -600,7 +600,7 @@ func (gm *GlobalModule) searchGlobalBatchHooks(hooksDir string) ([]*kind.BatchHo for idx, cfg := range sdkcfgs { nestedHookName := fmt.Sprintf("%s-%s-%d", hookName, cfg.Metadata.Name, idx) - shHook := kind.NewBatchHook(nestedHookName, hookPath, uint(idx), gm.keepTemporaryHookFiles, false, gm.logger.Named("batch-hook")) + shHook := kind.NewBatchHook(nestedHookName, hookPath, "global", uint(idx), gm.keepTemporaryHookFiles, false, gm.logger.Named("batch-hook")) hks = append(hks, shHook) } diff --git a/pkg/module_manager/module_manager.go b/pkg/module_manager/module_manager.go index 4c6e4495..2338cee9 100644 --- a/pkg/module_manager/module_manager.go +++ b/pkg/module_manager/module_manager.go @@ -18,6 +18,7 @@ import ( "github.com/flant/addon-operator/pkg/helm_resources_manager" . "github.com/flant/addon-operator/pkg/hook/types" "github.com/flant/addon-operator/pkg/kube_config_manager/config" + environmentmanager "github.com/flant/addon-operator/pkg/module_manager/environment_manager" gohook "github.com/flant/addon-operator/pkg/module_manager/go_hook" "github.com/flant/addon-operator/pkg/module_manager/loader" "github.com/flant/addon-operator/pkg/module_manager/loader/fs" @@ -71,6 +72,7 @@ type DirectoryConfig struct { ModulesDir string GlobalHooksDir string TempDir string + ChrootDir string } type KubeConfigManager interface { @@ -131,6 +133,8 @@ type ModuleManager struct { moduleScheduler *scheduler.Scheduler + environmentManager *environmentmanager.Manager + logger *log.Logger } @@ -143,7 +147,7 @@ func NewModuleManager(ctx context.Context, cfg *ModuleManagerConfig, logger *log // default loader, maybe we can register another one on startup fsLoader := fs.NewFileSystemLoader(cfg.DirectoryConfig.ModulesDir, logger.Named("file-system-loader")) - return &ModuleManager{ + mm := &ModuleManager{ ctx: cctx, cancel: cancel, @@ -165,6 +169,19 @@ func NewModuleManager(ctx context.Context, cfg *ModuleManagerConfig, logger *log logger: logger, } + + if len(cfg.DirectoryConfig.ChrootDir) > 0 { + mm.environmentManager = environmentmanager.NewManager(cfg.DirectoryConfig.ChrootDir, mm.logger) + } + + return mm +} + +// AddObjectsToChrootEnvironment sets the list of objects to implement in the modules' chroot directories +func (mm *ModuleManager) AddObjectsToChrootEnvironment(objects ...environmentmanager.ObjectDescriptor) { + if mm.EnvironmentManagerEnabled() { + mm.environmentManager.AddObjectsToEnvironment(objects...) + } } func (mm *ModuleManager) Stop() { @@ -639,7 +656,7 @@ func (mm *ModuleManager) DeleteModule(moduleName string, logLabels map[string]st // Unregister module hooks. ml.DeregisterHooks() - return nil + return ml.DisassembleEnvironmentForModule() } // RunModule runs beforeHelm hook, helm upgrade --install and afterHelm or afterDeleteHelm hook @@ -1267,13 +1284,13 @@ func (mm *ModuleManager) registerModules(scriptEnabledExtender *script_extender. set := &moduleset.ModulesSet{} - // load and registry global hooks dep := &hooks.HookExecutionDependencyContainer{ HookMetricsStorage: mm.dependencies.HookMetricStorage, KubeConfigManager: mm.dependencies.KubeConfigManager, KubeObjectPatcher: mm.dependencies.KubeObjectPatcher, MetricStorage: mm.dependencies.MetricStorage, GlobalValuesGetter: mm.global, + EnvironmentManager: mm.environmentManager, } for _, mod := range mods { @@ -1285,12 +1302,12 @@ func (mm *ModuleManager) registerModules(scriptEnabledExtender *script_extender. } mod.WithDependencies(dep) - set.Add(mod) - err := mm.moduleScheduler.AddModuleVertex(mod) - if err != nil { - return err + + if err := mm.moduleScheduler.AddModuleVertex(mod); err != nil { + return fmt.Errorf("add module vertex: %w", err) } + scriptEnabledExtender.AddBasicModule(mod) mm.SendModuleEvent(events.ModuleEvent{ @@ -1332,6 +1349,10 @@ func (mm *ModuleManager) ModuleHasCRDs(moduleName string) bool { return mm.GetModule(moduleName).CRDExist() } +func (mm *ModuleManager) EnvironmentManagerEnabled() bool { + return mm.environmentManager != nil +} + // queueHasPendingModuleRunTaskWithStartup returns true if queue has pending tasks // with the type "ModuleRun" related to the module "moduleName" and DoModuleStartup is set to true. func queueHasPendingModuleRunTaskWithStartup(q *queue.TaskQueue, moduleName string) bool { diff --git a/pkg/module_manager/scheduler/extenders/script_enabled/script.go b/pkg/module_manager/scheduler/extenders/script_enabled/script.go index 750261f2..b52bd8d2 100644 --- a/pkg/module_manager/scheduler/extenders/script_enabled/script.go +++ b/pkg/module_manager/scheduler/extenders/script_enabled/script.go @@ -17,6 +17,8 @@ import ( utils_file "github.com/flant/shell-operator/pkg/utils/file" ) +type scriptState string + const ( Name extenders.ExtenderName = "ScriptEnabled" @@ -25,8 +27,6 @@ const ( statError scriptState = "StatError" ) -type scriptState string - type Extender struct { tmpDir string basicModuleDescriptors map[string]moduleDescriptor @@ -46,7 +46,7 @@ func NewExtender(tmpDir string) (*Extender, error) { } if !info.IsDir() { - return nil, fmt.Errorf("%s path isn't a directory", tmpDir) + return nil, fmt.Errorf("%q path isn't a directory", tmpDir) } e := &Extender{ @@ -81,6 +81,7 @@ func (e *Extender) AddBasicModule(module node.ModuleInterface) { slog.String("module", module.GetName())) } } + e.basicModuleDescriptors[module.GetName()] = moduleD } diff --git a/pkg/module_manager/scheduler/extenders/static/static.go b/pkg/module_manager/scheduler/extenders/static/static.go index f275e490..d11dcf6d 100644 --- a/pkg/module_manager/scheduler/extenders/static/static.go +++ b/pkg/module_manager/scheduler/extenders/static/static.go @@ -29,7 +29,7 @@ func NewExtender(staticValuesFilePaths string) (*Extender, error) { valuesFile := filepath.Join(dir, "values.yaml") fileInfo, err := os.Stat(valuesFile) if err != nil { - log.Error("Couldn't stat file", + log.Warn("Couldn't stat file", slog.String("file", valuesFile)) continue } diff --git a/pkg/utils/chroot.go b/pkg/utils/chroot.go new file mode 100644 index 00000000..baf4d242 --- /dev/null +++ b/pkg/utils/chroot.go @@ -0,0 +1,15 @@ +package utils + +import ( + "fmt" + + "github.com/flant/addon-operator/pkg/app" +) + +func GetModuleChrootPath(moduleName string) string { + if len(app.ShellChrootDir) > 0 { + return fmt.Sprintf("%s/%s", app.ShellChrootDir, moduleName) + } + + return "" +}