Move env plugin to builtins

backends/client.go

@@ -4,30 +4,59 @@ import (
 	"os/exec"
 
 	plugin "github.com/hashicorp/go-plugin"
-	"github.com/kelseyhightower/confd/backends/commons"
+	"github.com/kelseyhightower/confd/confd"
 	"github.com/kelseyhightower/confd/log"
+	confdplugin "github.com/kelseyhightower/confd/plugin"
 )
 
-// handshakeConfigs are used to just do a basic handshake between
-// a plugin and host. If the handshake fails, a user friendly error is shown.
-// This prevents users from executing bad plugins or executing a plugin
-// directory. It is a UX feature, not a security feature.
-var handshakeConfig = plugin.HandshakeConfig{
-	ProtocolVersion:  1,
-	MagicCookieKey:   "BASIC_PLUGIN",
-	MagicCookieValue: "hello",
-}
-
-// pluginMap is the map of plugins we can dispense.
-var pluginMap = map[string]plugin.Plugin{
-	"env": &commons.StoreClientPlugin{},
-}
-
 // New is used to create a storage client based on our configuration.
-func New(config Config) (commons.StoreClient, error) {
+func New(config Config) (confd.Database, error) {
+	// if config.Backend == "" {
+	// 	config.Backend = "etcd"
+	// }
+	// backendNodes := config.BackendNodes
+	// log.Info("Backend nodes set to " + strings.Join(backendNodes, ", "))
+	// switch config.Backend {
+	// case "consul":
+	// 	return consul.New(config.BackendNodes, config.Scheme,
+	// 		config.ClientCert, config.ClientKey,
+	// 		config.ClientCaKeys)
+	// case "etcd":
+	// 	// Create the etcd client upfront and use it for the life of the process.
+	// 	// The etcdClient is an http.Client and designed to be reused.
+	// 	return etcd.NewEtcdClient(backendNodes, config.ClientCert, config.ClientKey, config.ClientCaKeys, config.BasicAuth, config.Username, config.Password)
+	// case "zookeeper":
+	// 	return zookeeper.NewZookeeperClient(backendNodes)
+	// case "rancher":
+	// 	return rancher.NewRancherClient(backendNodes)
+	// case "redis":
+	// 	return redis.NewRedisClient(backendNodes, config.ClientKey)
+	// case "env":
+	// 	return env.NewEnvClient()
+	// case "vault":
+	// 	vaultConfig := map[string]string{
+	// 		"app-id":   config.AppID,
+	// 		"user-id":  config.UserID,
+	// 		"username": config.Username,
+	// 		"password": config.Password,
+	// 		"token":    config.AuthToken,
+	// 		"cert":     config.ClientCert,
+	// 		"key":      config.ClientKey,
+	// 		"caCert":   config.ClientCaKeys,
+	// 	}
+	// 	return vault.New(backendNodes[0], config.AuthType, vaultConfig)
+	// case "dynamodb":
+	// 	table := config.Table
+	// 	log.Info("DynamoDB table set to " + table)
+	// 	return dynamodb.NewDynamoDBClient(table)
+	// case "stackengine":
+	// 	return stackengine.NewStackEngineClient(backendNodes, config.Scheme, config.ClientCert, config.ClientKey, config.ClientCaKeys, config.AuthToken)
+	// }
+	// return nil, errors.New("Invalid backend")
+
 	client := plugin.NewClient(&plugin.ClientConfig{
-		HandshakeConfig: handshakeConfig,
-		Plugins:         pluginMap,
+		HandshakeConfig: confdplugin.HandshakeConfig,
+		Plugins:         confdplugin.PluginMap,
 		Cmd:             exec.Command("/Users/oleksa/go/src/github.com/kelseyhightower/confd/bin/plugins"),
 	})
 	defer client.Kill()
@@ -44,7 +73,5 @@ func New(config Config) (commons.StoreClient, error) {
 		log.Fatal(err.Error())
 	}
 
-	// We should have a Greeter now! This feels like a normal interface
-	// implementation but is in fact over an RPC connection.
-	return raw.(commons.StoreClient), nil
+	return raw.(confd.Database), nil
 }

builtin/bins/env/main.go

@@ -0,0 +1,9 @@
+package main
+
+import "github.com/hashicorp/terraform/plugin"
+
+func main() {
+	plugin.Serve(&plugin.ServeOpts{
+		DatabaseFunc: env.Database,
+	})
+}

backends/env/plugin/plugin.go → builtin/databases/env/database.go

@@ -5,7 +5,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/kelseyhightower/confd/backends/env/util"
+	"github.com/kelseyhightower/confd/builtin/databases/env/util"
 	"github.com/kelseyhightower/confd/log"
 )
 

builtin/databases/env/main.go

@@ -0,0 +1,31 @@
+package env
+
+// package main
+//
+// import (
+// 	plugin "github.com/hashicorp/go-plugin"
+// 	"github.com/kelseyhightower/confd/backends/commons"
+// 	env "github.com/kelseyhightower/confd/backends/env/plugin"
+// )
+//
+// // handshakeConfigs are used to just do a basic handshake between
+// // a plugin and host. If the handshake fails, a user friendly error is shown.
+// // This prevents users from executing bad plugins or executing a plugin
+// // directory. It is a UX feature, not a security feature.
+// var handshakeConfig = plugin.HandshakeConfig{
+// 	ProtocolVersion:  1,
+// 	MagicCookieKey:   "BASIC_PLUGIN",
+// 	MagicCookieValue: "hello",
+// }
+//
+// // pluginMap is the map of plugins we can dispense.
+// var pluginMap = map[string]plugin.Plugin{
+// 	"env": &commons.DatabasePlugin{Impl: new(env.Client)},
+// }
+//
+// func main() {
+// 	plugin.Serve(&plugin.ServeConfig{
+// 		HandshakeConfig: handshakeConfig,
+// 		Plugins:         pluginMap,
+// 	})
+// }

backends/env/util/util.go → builtin/databases/env/util/util.go

@@ -2,16 +2,14 @@ package util
 
 import "strings"
 
-var replacer = strings.NewReplacer("/", "_")
-
 func Transform(key string) string {
+	replacer := strings.NewReplacer("/", "_")
 	k := strings.TrimPrefix(key, "/")
 	return strings.ToUpper(replacer.Replace(k))
 }
 
-var cleanReplacer = strings.NewReplacer("_", "/")
-
 func Clean(key string) string {
+	cleanReplacer := strings.NewReplacer("_", "/")
 	newKey := "/" + key
 	return cleanReplacer.Replace(strings.ToLower(newKey))
 }

confd.go

@@ -23,12 +23,12 @@ func main() {
 
 	log.Info("Starting confd")
 
-	storeClient, err := backends.New(backendsConfig)
+	database, err := backends.New(backendsConfig)
 	if err != nil {
 		log.Fatal(err.Error())
 	}
 
-	templateConfig.StoreClient = storeClient
+	templateConfig.Database = database
 	if onetime {
 		if err := template.Process(templateConfig); err != nil {
 			log.Fatal(err.Error())

confd/database.go

@@ -0,0 +1,8 @@
+package confd
+
+// The Database interface is implemented by objects that can retrieve
+// key/value pairs from a backend store.
+type Database interface {
+	GetValues(keys []string) (map[string]string, error)
+	WatchPrefix(prefix string, keys []string, waitIndex uint64, stopChan chan bool) (uint64, error)
+}

docs/plugin-system.md

@@ -12,3 +12,9 @@ How plugins should be configured?
 I prefer number 2 for now, as it's easier to implement and maintain. But it can be changed later on.
 
 Plugin systems is implemented using https://github.com/hashicorp/go-plugin, which is successfully used in Terraform, Packer, Nomad and Vault. It's based on network RPC over local network. Plugins are Go interface implementations, which maps quite good to the current backends implementation.
+
+The core set of plugins is defined in builtin/ directory. They are compiled in the main binary.
+To add a new plugin:
+
+Builtin plugins are called using special cli command:
+confd internal-plugin <pluginType> <pluginName>

internal_plugin.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/kardianos/osext"
+	"github.com/kelseyhightower/confd/plugin"
+)
+
+const CONFDSPACE = "-CONFDSPACE-"
+
+// BuildPluginCommandString builds a special string for executing internal
+// plugins. It has the following format:
+//
+// 	/path/to/terraform-CONFDSPACE-internal-plugin-CONFDSPACE-terraform-provider-aws
+//
+// We split the string on -CONFDSPACE- to build the command executor. The reason we
+// use -CONFDSPACE- is so we can support spaces in the /path/to/terraform part.
+func BuildPluginCommandString(pluginType, pluginName string) (string, error) {
+	terraformPath, err := osext.Executable()
+	if err != nil {
+		return "", err
+	}
+	parts := []string{terraformPath, "internal-plugin", pluginType, pluginName}
+	return strings.Join(parts, CONFDSPACE), nil
+}
+
+func Run(args []string) int {
+	if len(args) != 2 {
+		log.Printf("Wrong number of args; expected: confd internal-plugin pluginType pluginName")
+		return 1
+	}
+
+	pluginType := args[0]
+	pluginName := args[1]
+
+	log.SetPrefix(fmt.Sprintf("%s-%s (internal) ", pluginName, pluginType))
+
+	switch pluginType {
+	case "database":
+		pluginFunc, found := InternalProviders[pluginName]
+		if !found {
+			log.Printf("[ERROR] Could not load provider: %s", pluginName)
+			return 1
+		}
+		log.Printf("[INFO] Starting provider plugin %s", pluginName)
+		plugin.Serve(&plugin.ServeOpts{
+			ProviderFunc: pluginFunc,
+		})
+	default:
+		log.Printf("[ERROR] Invalid plugin type %s", pluginType)
+		return 1
+	}
+
+	return 0
+}

internal_plugin_list.go

@@ -0,0 +1,10 @@
+package main
+
+import (
+	"github.com/kelseyhightower/confd/builtin/databases/env"
+	"github.com/kelseyhightower/confd/confd"
+)
+
+var InternalDatabases = map[string]confd.Database{
+	"env": &env.Client{},
+}