feat: support for external middleware plugins

This commit introduces possibility to integrate external middleware
plugins. These plugins have the possibility to modify incoming HTTP
requests prior to routing. This includes modifying headers, routing and
denying requests.

Due to the lack of a proper plugin system in Go, external middleware
plugins are implemented as standalone sub-processes. They communicate
with wfx via protobuf message over stdin/stdout, hence it's crucial for
the plugin to read from stdin promptly to avoid blocking wfx. An example
plugin written in Go is included in the `example/plugin` subdirectory,
taking special care that stdin does not block.

Signed-off-by: Michael Adler <[email protected]>

.github/workflows/ci.yml

@@ -73,7 +73,7 @@ jobs:
           --health-retries 20
     steps:
       - uses: actions/checkout@v4
-      - run: go test -timeout 180s -race -coverprofile=coverage.out -covermode=atomic -tags testing,integration,postgres,sqlite ./...
+      - run: go test -timeout 180s -race -coverprofile=coverage.out -covermode=atomic -tags testing,integration,postgres,sqlite,plugin ./...
         env:
           PGHOST: postgres
           PGPORT: 5432
@@ -109,7 +109,7 @@ jobs:
           --health-retries 20
     steps:
       - uses: actions/checkout@v4
-      - run: go test -timeout 180s -race -coverprofile=coverage.out -covermode=atomic -tags testing,integration,mysql,sqlite ./...
+      - run: go test -timeout 180s -race -coverprofile=coverage.out -covermode=atomic -tags testing,integration,mysql,sqlite,plugin ./...
         env:
           MYSQL_DATABASE: wfx
           MYSQL_ROOT_PASSWORD: root
@@ -196,22 +196,23 @@ jobs:
       - uses: dominikh/[email protected]
         with:
           install-go: false
-          build-tags: sqlite,testing
+          build-tags: sqlite,testing,plugin
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: latest
-          args: --build-tags=sqlite,testing
+          args: --build-tags=sqlite,testing,plugin
           skip-cache: true
 
   generate:
     name: Generate Code
     runs-on: ubuntu-latest
     container:
-      image: quay.io/goswagger/swagger
+      image: archlinux
     steps:
+      - name: Install packages
+        run: pacman -Syu --noconfirm python-yaml git just go flatbuffers go-swagger gofumpt
       - uses: actions/checkout@v4
-      - run: apk add --no-cache py3-yaml git just bash go
       - name: Disable git security features
         run: git config --global safe.directory '*'
       - uses: brokeyourbike/go-mockery-action@v0

.gitlab-ci.yml

@@ -60,10 +60,9 @@ generate:
   stage: lint
   needs: []
   image:
-    name: quay.io/goswagger/swagger
-    entrypoint: [""] # needed to get a shell
+    name: archlinux:latest
   before_script:
-    - apk add --no-cache py3-yaml git just bash go git-lfs
+    - pacman -Syu --noconfirm python-yaml git just go flatbuffers go-swagger gofumpt
     - git lfs install && git submodule update
   script:
     - just generate

.golangci.yml

@@ -88,6 +88,7 @@ linters:
     - tparallel
     - usestdlibvars
     - wrapcheck
+    - zerologlint
 
 linters-settings:
   staticcheck:
@@ -101,3 +102,5 @@ linters-settings:
   wrapcheck:
     ignorePackageGlobs:
       - github.com/siemens/wfx/internal/errutil
+      - google.golang.org/protobuf/*
+      - io

.goreleaser.yml

@@ -18,6 +18,7 @@ builds:
       - sqlite
       - postgres
       - mysql
+      - plugin
     flags:
       - -trimpath
       - -mod=readonly

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add optional `description` field to workflows
 - Job event notifications via server-sent events (see #11)
+- Plugin System for External Middlewares (see #43)
 
 ### Fixed
 

Makefile

@@ -13,7 +13,7 @@ MAKEFLAGS += --jobs=$(shell nproc)
 DESTDIR ?=
 prefix ?= /usr/local
 
-GO_TAGS = sqlite,postgres,mysql
+GO_TAGS = sqlite,postgres,mysql,plugin
 
 export CGO_ENABLED=1
 
@@ -31,7 +31,7 @@ default:
 
 .PHONY: test
 test:
-	go test -race -coverprofile=coverage.out -covermode=atomic -timeout 30s ./... "--tags=sqlite,testing"
+	go test -race -coverprofile=coverage.out -covermode=atomic -timeout 30s ./... "--tags=sqlite,testing,plugin"
 
 .PHONY: install
 install:

cmd/wfx/cmd/root/cmd_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
 	"github.com/siemens/wfx/cmd/wfxctl/flags"
+	"github.com/siemens/wfx/persistence"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -225,3 +226,27 @@ func waitForLogLevel(t *testing.T, expected zerolog.Level) {
 	}
 	require.Equal(t, expected.String(), zerolog.GlobalLevel().String())
 }
+
+func TestCreateNorthboundCollection_PluginsDir(t *testing.T) {
+	dir, _ := os.MkdirTemp("", "TestCreateNorthboundCollection_PluginsDir.*")
+	k.Write(func(k *koanf.Koanf) {
+		_ = k.Set(mgmtPluginsDirFlag, dir)
+	})
+	dbMock := persistence.NewMockStorage(t)
+	sc, err := createNorthboundCollection([]string{"http"}, dbMock)
+	t.Cleanup(func() { sc.Shutdown(context.Background()) })
+	assert.NoError(t, err)
+	assert.NotNil(t, sc)
+}
+
+func TestCreateSouthboundCollection_PluginsDir(t *testing.T) {
+	dir, _ := os.MkdirTemp("", "TestCreateSouthboundCollection_PluginsDir.*")
+	k.Write(func(k *koanf.Koanf) {
+		_ = k.Set(clientPluginsDirFlag, dir)
+	})
+	dbMock := persistence.NewMockStorage(t)
+	sc, err := createSouthboundCollection([]string{"http"}, dbMock)
+	t.Cleanup(func() { sc.Shutdown(context.Background()) })
+	assert.NoError(t, err)
+	assert.NotNil(t, sc)
+}

cmd/wfx/cmd/root/flags.go

@@ -60,6 +60,10 @@ const (
 
 	preferedStorage    = "sqlite"
 	defaultStorageOpts = "file:wfx.db?_fk=1&_journal=WAL"
+
+	// Plugins
+	clientPluginsDirFlag = "client-plugins-dir"
+	mgmtPluginsDirFlag   = "mgmt-plugins-dir"
 )
 
 func init() {
@@ -109,6 +113,13 @@ func init() {
 	f.StringSlice(configFlag, config.DefaultConfigFiles(), "path to one or more .yaml config files")
 	_ = Command.MarkPersistentFlagFilename(configFlag, "yml", "yaml")
 
+	// plugins
+	_ = Command.MarkPersistentFlagDirname(clientPluginsDirFlag)
+	f.String(clientPluginsDirFlag, "", "directory containing client plugins")
+
+	_ = Command.MarkPersistentFlagDirname(mgmtPluginsDirFlag)
+	f.String(mgmtPluginsDirFlag, "", "directory containing management plugins")
+
 	{
 		var defaultStorage string
 		supportedStorages := persistence.Storages()

cmd/wfx/cmd/root/northbound.go

@@ -27,12 +27,15 @@ import (
 
 func createNorthboundCollection(schemes []string, storage persistence.Storage) (*serverCollection, error) {
 	var settings server.HTTPSettings
+	var pluginsDir string
 	k.Read(func(k *koanf.Koanf) {
 		settings.Host = k.String(mgmtHostFlag)
 		settings.TLSHost = k.String(mgmtTLSHostFlag)
 		settings.Port = k.Int(mgmtPortFlag)
 		settings.TLSPort = k.Int(mgmtTLSPortFlag)
 		settings.UDSPath = k.String(mgmtUnixSocketFlag)
+
+		pluginsDir = k.String(mgmtPluginsDirFlag)
 	})
 	api := api.NewNorthboundAPI(storage)
 	fsMW, err := fileserver.NewFileServerMiddleware(k)
@@ -41,18 +44,27 @@ func createNorthboundCollection(schemes []string, storage persistence.Storage) (
 	}
 
 	swaggerJSON, _ := restapi.SwaggerJSON.MarshalJSON()
-	mw := middleware.NewGlobalMiddleware(restapi.ConfigureAPI(api),
-		[]middleware.IntermediateMW{
-			// LIFO
-			logging.MW{},
-			jq.MW{},
-			fsMW,
-			swagger.NewSpecMiddleware(api.Context().BasePath(), swaggerJSON),
-			health.NewHealthMiddleware(storage),
-			version.MW{},
-			middleware.PromoteWrapper(cors.AllowAll().Handler),
-		})
 
+	// LIFO, i.e. middlewares are applied in reverse order
+	intermdiateMws := []middleware.IntermediateMW{
+		jq.MW{},
+		fsMW,
+		swagger.NewSpecMiddleware(api.Context().BasePath(), swaggerJSON),
+		health.NewHealthMiddleware(storage),
+		version.MW{},
+		middleware.PromoteWrapper(cors.AllowAll().Handler),
+	}
+
+	if pluginsDir != "" {
+		mws, err := createPluginMiddlewares(pluginsDir)
+		if err != nil {
+			return nil, fault.Wrap(err)
+		}
+		intermdiateMws = append(intermdiateMws, mws...)
+	}
+	intermdiateMws = append(intermdiateMws, logging.MW{})
+
+	mw := middleware.NewGlobalMiddleware(restapi.ConfigureAPI(api), intermdiateMws)
 	servers, err := createServers(schemes, mw, settings)
 	if err != nil {
 		return nil, fault.Wrap(err)

cmd/wfx/cmd/root/plugins.go

@@ -0,0 +1,68 @@
+package root
+
+/*
+ * SPDX-FileCopyrightText: 2023 Siemens AG
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Author: Michael Adler <[email protected]>
+ */
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+
+	"github.com/Southclaws/fault"
+	"github.com/rs/zerolog/log"
+	"github.com/siemens/wfx/middleware"
+	"github.com/siemens/wfx/middleware/plugin"
+)
+
+func createPluginMiddlewares(pluginsDir string) ([]middleware.IntermediateMW, error) {
+	pluginMws, err := loadPlugins(pluginsDir)
+	if err != nil {
+		return nil, fault.Wrap(err)
+	}
+	result := make([]middleware.IntermediateMW, 0, len(pluginMws))
+	for _, p := range pluginMws {
+		mw, err := plugin.NewMiddleware(p)
+		if err != nil {
+			return nil, fault.Wrap(err)
+		}
+		result = append(result, mw)
+	}
+	return result, nil
+}
+
+func loadPlugins(dir string) ([]plugin.Plugin, error) {
+	log.Debug().Msg("Loading plugins")
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fault.Wrap(err)
+	}
+
+	result := make([]plugin.Plugin, 0, len(entries))
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			dest, err := filepath.EvalSymlinks(path.Join(dir, entry.Name()))
+			if err != nil {
+				return nil, fault.Wrap(err)
+			}
+			info, err := os.Stat(dest)
+			if err != nil {
+				return nil, fault.Wrap(err)
+			}
+			// check if file is executable
+			if (info.Mode() & 0o111) != 0 {
+				result = append(result, plugin.NewFBPlugin(dest))
+			} else {
+				log.Warn().Str("dest", dest).Msg("Ignoring non-executable file")
+			}
+		}
+	}
+	sort.Slice(result, func(i int, j int) bool { return result[i].Name() < result[j].Name() })
+	log.Debug().Int("count", len(result)).Msg("Loaded plugins")
+	return result, nil
+}