From 4a8bd93ce5feeeae46d780069ec413bf1a4523dd Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 15:32:17 +0100 Subject: [PATCH 01/11] Add support for custom unmarshaling of strings Implemented an interface for custom unmarshaling of strings which allows users to define their own custom type unmarshaling methods. Updated fig_test.go and fig.go to reflect these changes. This update provides a flexible way for users to handle configs with custom types. --- fig.go | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++ fig_test.go | 53 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 116 insertions(+), 4 deletions(-) diff --git a/fig.go b/fig.go index 4f6232a..6fbb003 100644 --- a/fig.go +++ b/fig.go @@ -28,6 +28,42 @@ const ( DefaultTimeLayout = time.RFC3339 ) +// StringUnmarshaler is an interface for custom unmarshaling of strings +// +// If a field with a local type asignment satisfies this interface, it allows the user +// to implment their own custom type unmarshaling method. +// +// Example: +// +// type ListenerType uint +// +// const ( +// ListenerUnix ListenerType = iota +// ListenerTCP +// ListenerTLS +// ) +// +// type Config struct { +// Listener ListenerType `fig:"listener_type" default:"unix"` +// } +// +// func (l *ListenerType) UnmarshalType(v string) error { +// switch strings.ToLower(v) { +// case "unix": +// *l = ListenerUnix +// case "tcp": +// *l = ListenerTCP +// case "tls": +// *l = ListenerTLS +// default: +// return fmt.Errorf("unknown listener type: %s", v) +// } +// return nil +// } +type StringUnmarshaler interface { + UnmarshalString(s string) error +} + // Load reads a configuration file and loads it into the given struct. The // parameter `cfg` must be a pointer to a struct. // @@ -158,6 +194,7 @@ func (f *fig) decodeMap(m map[string]interface{}, result interface{}) error { mapstructure.StringToTimeDurationHookFunc(), mapstructure.StringToTimeHookFunc(f.timeLayout), stringToRegexpHookFunc(), + stringToStringUnmarshalerHook(), ), }) if err != nil { @@ -183,6 +220,36 @@ func stringToRegexpHookFunc() mapstructure.DecodeHookFunc { } } +// stringToStringUnmarshalerHook returns a DecodeHookFunc that executes a custom method which +// satisfies the StringUnmarshaler interface on custom types. +func stringToStringUnmarshalerHook() mapstructure.DecodeHookFunc { + return func(f reflect.Type, t reflect.Type, data interface{}) (interface{}, error) { + if f.Kind() != reflect.String { + return data, nil + } + + ds, ok := data.(string) + if !ok { + return data, nil + } + + if reflect.PointerTo(t).Implements(reflect.TypeOf((*StringUnmarshaler)(nil)).Elem()) { + val := reflect.New(t).Interface() + + if unmarshaler, ok := val.(StringUnmarshaler); ok { + err := unmarshaler.UnmarshalString(ds) + if err != nil { + return nil, err + } + + return reflect.ValueOf(val).Elem().Interface(), nil + } + } + + return data, nil + } +} + // processCfg processes a cfg struct after it has been loaded from // the config file, by validating required fields and setting defaults // where applicable. diff --git a/fig_test.go b/fig_test.go index 737ab64..3811ec0 100644 --- a/fig_test.go +++ b/fig_test.go @@ -84,6 +84,14 @@ type Item struct { Path string `fig:"path" validate:"required"` } +type ListenerType uint + +const ( + ListenerUnix ListenerType = iota + ListenerTCP + ListenerTLS +) + func validPodConfig() Pod { var pod Pod @@ -249,6 +257,9 @@ func Test_fig_Load_Defaults(t *testing.T) { Application struct { BuildDate time.Time `fig:"build_date" default:"2020-01-01T12:00:00Z"` } + Server struct { + Listener ListenerType `fig:"listener_type" default:"unix"` + } } var want Server @@ -259,6 +270,7 @@ func Test_fig_Load_Defaults(t *testing.T) { want.Logger.Production = false want.Logger.Metadata.Keys = []string{"ts"} want.Application.BuildDate = time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) + want.Server.Listener = ListenerTCP var cfg Server err := Load(&cfg, File(f), Dirs(filepath.Join("testdata", "valid"))) @@ -590,8 +602,9 @@ func Test_fig_decodeMap(t *testing.T) { "log_level": "debug", "severity": "5", "server": map[string]interface{}{ - "ports": []int{443, 80}, - "secure": 1, + "ports": []int{443, 80}, + "secure": 1, + "listener_type": "tls", }, } @@ -599,8 +612,9 @@ func Test_fig_decodeMap(t *testing.T) { Level string `fig:"log_level"` Severity int `fig:"severity" validate:"required"` Server struct { - Ports []string `fig:"ports" default:"[443]"` - Secure bool + Ports []string `fig:"ports" default:"[443]"` + Secure bool + Listener ListenerType `fig:"listener_type" default:"unix"` } `fig:"server"` } @@ -623,6 +637,10 @@ func Test_fig_decodeMap(t *testing.T) { if cfg.Server.Secure == false { t.Error("cfg.Server.Secure == false") } + + if cfg.Server.Listener != ListenerTLS { + t.Errorf("cfg.Server.Listener: want: %s, got: %s", ListenerTLS, cfg.Server.Listener) + } } func Test_fig_processCfg(t *testing.T) { @@ -1263,3 +1281,30 @@ func setenv(t *testing.T, key, value string) { t.Helper() t.Setenv(key, value) } + +func (l *ListenerType) UnmarshalString(v string) error { + switch strings.ToLower(v) { + case "unix": + *l = ListenerUnix + case "tcp": + *l = ListenerTCP + case "tls": + *l = ListenerTLS + default: + return fmt.Errorf("unknown listener type: %s", v) + } + return nil +} + +func (l ListenerType) String() string { + switch l { + case ListenerUnix: + return "unix" + case ListenerTCP: + return "tcp" + case ListenerTLS: + return "tls" + default: + return "unknown" + } +} From e3a68337f8047eff3c1816c11a63883dc68fb19f Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 19:02:39 +0100 Subject: [PATCH 02/11] Add "listener_type" to server configuration Added "listener_type" field to the server configuration in JSON, YAML, and TOML files. The new field helps initialising the ListenerType field, which is now outside the Server struct in the fig test go file, with the "tcp" value. --- testdata/valid/server.json | 3 ++- testdata/valid/server.toml | 3 ++- testdata/valid/server.yaml | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/testdata/valid/server.json b/testdata/valid/server.json index e7edd71..c0d4d19 100644 --- a/testdata/valid/server.json +++ b/testdata/valid/server.json @@ -2,5 +2,6 @@ "host": "0.0.0.0", "logger": { "log_level": "debug" - } + }, + "listener_type": "tcp" } \ No newline at end of file diff --git a/testdata/valid/server.toml b/testdata/valid/server.toml index a5ba440..22ec5eb 100644 --- a/testdata/valid/server.toml +++ b/testdata/valid/server.toml @@ -1,4 +1,5 @@ host = "0.0.0.0" +listener_type = "tcp" [logger] -log_level = "debug" \ No newline at end of file +log_level = "debug" diff --git a/testdata/valid/server.yaml b/testdata/valid/server.yaml index c6ddab8..03268ee 100644 --- a/testdata/valid/server.yaml +++ b/testdata/valid/server.yaml @@ -1,4 +1,5 @@ host: "0.0.0.0" +listener_type: "tcp" logger: log_level: "debug" \ No newline at end of file From 97b2ee42c5b5b45ea61ebb286846c1c4db0d6481 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 19:03:00 +0100 Subject: [PATCH 03/11] Refactor listener placement in fig test The "listener_type" field has moved out of the Server struct and is now directly under the server configuration in the fig test go file. This change simplifies the initialization of the ListenerType field with the "tcp" value in JSON, YAML, and TOML configuration files. --- fig_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fig_test.go b/fig_test.go index 3811ec0..2bf1ef5 100644 --- a/fig_test.go +++ b/fig_test.go @@ -257,9 +257,7 @@ func Test_fig_Load_Defaults(t *testing.T) { Application struct { BuildDate time.Time `fig:"build_date" default:"2020-01-01T12:00:00Z"` } - Server struct { - Listener ListenerType `fig:"listener_type" default:"unix"` - } + Listener ListenerType `fig:"listener_type" default:"unix"` } var want Server @@ -270,7 +268,7 @@ func Test_fig_Load_Defaults(t *testing.T) { want.Logger.Production = false want.Logger.Metadata.Keys = []string{"ts"} want.Application.BuildDate = time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) - want.Server.Listener = ListenerTCP + want.Listener = ListenerTCP var cfg Server err := Load(&cfg, File(f), Dirs(filepath.Join("testdata", "valid"))) From 874e37dd248de210d18b4067c9d76fffee8d5761 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 19:37:35 +0100 Subject: [PATCH 04/11] Update setDefaultValue function in fig.go The function setDefaultValue in fig.go has been modified to call setValue unless the value satisfies the StringUnmarshaler interface. --- fig.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/fig.go b/fig.go index 6fbb003..9eb8b8d 100644 --- a/fig.go +++ b/fig.go @@ -313,12 +313,23 @@ func (f *fig) formatEnvKey(key string) string { return strings.ToUpper(key) } -// setDefaultValue calls setValue but disallows booleans from -// being set. +// setDefaultValue calls setValue unless the value satisfies the +// StringUnmarshaler interface or is of a boolean type. func (f *fig) setDefaultValue(fv reflect.Value, val string) error { if fv.Kind() == reflect.Bool { return fmt.Errorf("unsupported type: %v", fv.Kind()) } + if reflect.PointerTo(fv.Type()).Implements(reflect.TypeOf((*StringUnmarshaler)(nil)).Elem()) { + vi := reflect.New(fv.Type()).Interface() + if unmarshaler, ok := vi.(StringUnmarshaler); ok { + err := unmarshaler.UnmarshalString(val) + if err != nil { + return err + } + fv.Set(reflect.ValueOf(vi).Elem()) + } + return nil + } return f.setValue(fv, val) } From 04d9b3c6047b747d1e6a4e355ecda8cf687d4758 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 19:38:34 +0100 Subject: [PATCH 05/11] Add custom configuration and test files A new custom configuration file, config.yaml, and a corresponding test file, custom_test.go, have been created. This is to serve as example for the custom UnmarshalString interface --- examples/custom/config.yaml | 7 ++++ examples/custom/custom_test.go | 73 ++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 examples/custom/config.yaml create mode 100644 examples/custom/custom_test.go diff --git a/examples/custom/config.yaml b/examples/custom/config.yaml new file mode 100644 index 0000000..96713bf --- /dev/null +++ b/examples/custom/config.yaml @@ -0,0 +1,7 @@ +app: + environment: dev + +server: + port: 443 + read_timeout: 1m + diff --git a/examples/custom/custom_test.go b/examples/custom/custom_test.go new file mode 100644 index 0000000..6054d02 --- /dev/null +++ b/examples/custom/custom_test.go @@ -0,0 +1,73 @@ +package custom + +import ( + "fmt" + "strings" + + "github.com/kkyr/fig" +) + +type ListenerType uint + +const ( + ListenerUnix ListenerType = iota + ListenerTCP + ListenerTLS +) + +type Config struct { + App struct { + Environment string `fig:"environment" validate:"required"` + } `fig:"app"` + Server struct { + Host string `fig:"host" default:"0.0.0.0"` + Port int `fig:"port" default:"80"` + Listener ListenerType `fig:"listener_type" default:"tcp"` + } `fig:"server"` +} + +func ExampleLoad() { + var cfg Config + err := fig.Load(&cfg) + if err != nil { + panic(err) + } + + fmt.Println(cfg.App.Environment) + fmt.Println(cfg.Server.Host) + fmt.Println(cfg.Server.Port) + fmt.Println(cfg.Server.Listener) + + // Output: + // dev + // 0.0.0.0 + // 443 + // tcp +} + +func (l *ListenerType) UnmarshalString(v string) error { + switch strings.ToLower(v) { + case "unix": + *l = ListenerUnix + case "tcp": + *l = ListenerTCP + case "tls": + *l = ListenerTLS + default: + return fmt.Errorf("unknown listener type: %s", v) + } + return nil +} + +func (l ListenerType) String() string { + switch l { + case ListenerUnix: + return "unix" + case ListenerTCP: + return "tcp" + case ListenerTLS: + return "tls" + default: + return "unknown" + } +} From ab1214d2fe978fadb6c7f945f70f472ae2ea799b Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Sun, 10 Dec 2023 20:02:46 +0100 Subject: [PATCH 06/11] Add ListenerType in Server struct in fig_test.go This is to make sure that `Test_fig_Load_UseStrict` won't fail due to the previously changed server.* files in `testdata/` --- fig_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fig_test.go b/fig_test.go index 2bf1ef5..92ea75f 100644 --- a/fig_test.go +++ b/fig_test.go @@ -375,7 +375,8 @@ func Test_fig_Load_UseStrict(t *testing.T) { for _, f := range []string{"server.yaml", "server.json", "server.toml"} { t.Run(f, func(t *testing.T) { type Server struct { - Host string `fig:"host"` + Host string `fig:"host"` + Listener ListenerType `fig:"listener_type"` } var cfg Server From 364c40f0a32fc72523dfaa837947ebc41655e629 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Mon, 11 Dec 2023 09:42:32 +0100 Subject: [PATCH 07/11] Moved string unmarshaling from setDefaultValue to setValue This is so that setEnv can benefit from that functionality as well --- fig.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/fig.go b/fig.go index 9eb8b8d..d3a2d13 100644 --- a/fig.go +++ b/fig.go @@ -313,12 +313,22 @@ func (f *fig) formatEnvKey(key string) string { return strings.ToUpper(key) } -// setDefaultValue calls setValue unless the value satisfies the -// StringUnmarshaler interface or is of a boolean type. +// setDefaultValue calls setValue but disallows booleans from +// being set. func (f *fig) setDefaultValue(fv reflect.Value, val string) error { if fv.Kind() == reflect.Bool { return fmt.Errorf("unsupported type: %v", fv.Kind()) } + return f.setValue(fv, val) +} + +// setValue sets fv to val. it attempts to convert val to the correct +// type based on the field's kind. if conversion fails an error is +// returned. If fv satisfies the StringUnmarshaler interface it will +// execute the corresponding StringUnmarshaler.UnmarshalString method +// on the value. +// fv must be settable else this panics. +func (f *fig) setValue(fv reflect.Value, val string) error { if reflect.PointerTo(fv.Type()).Implements(reflect.TypeOf((*StringUnmarshaler)(nil)).Elem()) { vi := reflect.New(fv.Type()).Interface() if unmarshaler, ok := vi.(StringUnmarshaler); ok { @@ -330,14 +340,7 @@ func (f *fig) setDefaultValue(fv reflect.Value, val string) error { } return nil } - return f.setValue(fv, val) -} -// setValue sets fv to val. it attempts to convert val to the correct -// type based on the field's kind. if conversion fails an error is -// returned. -// fv must be settable else this panics. -func (f *fig) setValue(fv reflect.Value, val string) error { switch fv.Kind() { case reflect.Ptr: if fv.IsNil() { From 48acdfe4406e9c613b8d230750e2a22b834cd8c8 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Mon, 11 Dec 2023 10:06:41 +0100 Subject: [PATCH 08/11] Add validity check and error wrapping in setValue function Added a check to ensure the reflect.Value is valid before attempting unmarshal. Also, wrapped error message for failed unmarshalling for clearer debugging. These changes will enhance error handling and debugging. --- fig.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fig.go b/fig.go index d3a2d13..7c60395 100644 --- a/fig.go +++ b/fig.go @@ -329,12 +329,12 @@ func (f *fig) setDefaultValue(fv reflect.Value, val string) error { // on the value. // fv must be settable else this panics. func (f *fig) setValue(fv reflect.Value, val string) error { - if reflect.PointerTo(fv.Type()).Implements(reflect.TypeOf((*StringUnmarshaler)(nil)).Elem()) { + if fv.IsValid() && reflect.PointerTo(fv.Type()).Implements(reflect.TypeOf((*StringUnmarshaler)(nil)).Elem()) { vi := reflect.New(fv.Type()).Interface() if unmarshaler, ok := vi.(StringUnmarshaler); ok { err := unmarshaler.UnmarshalString(val) if err != nil { - return err + return fmt.Errorf("could not unmarshal string %q: %w", val, err) } fv.Set(reflect.ValueOf(vi).Elem()) } From d4288aded4234792b937e92d030fc51551275c4c Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Mon, 11 Dec 2023 10:07:12 +0100 Subject: [PATCH 09/11] Remove 'listener_type' attribute from server configuration files again, following kkyr's suggestion --- testdata/valid/server.json | 3 +-- testdata/valid/server.toml | 1 - testdata/valid/server.yaml | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/testdata/valid/server.json b/testdata/valid/server.json index c0d4d19..e7edd71 100644 --- a/testdata/valid/server.json +++ b/testdata/valid/server.json @@ -2,6 +2,5 @@ "host": "0.0.0.0", "logger": { "log_level": "debug" - }, - "listener_type": "tcp" + } } \ No newline at end of file diff --git a/testdata/valid/server.toml b/testdata/valid/server.toml index 22ec5eb..ad60113 100644 --- a/testdata/valid/server.toml +++ b/testdata/valid/server.toml @@ -1,5 +1,4 @@ host = "0.0.0.0" -listener_type = "tcp" [logger] log_level = "debug" diff --git a/testdata/valid/server.yaml b/testdata/valid/server.yaml index 03268ee..c6ddab8 100644 --- a/testdata/valid/server.yaml +++ b/testdata/valid/server.yaml @@ -1,5 +1,4 @@ host: "0.0.0.0" -listener_type: "tcp" logger: log_level: "debug" \ No newline at end of file From 2889e9ee101c7768d086a046495fbb76c8f43d2d Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Mon, 11 Dec 2023 10:07:56 +0100 Subject: [PATCH 10/11] Repositioned UnmarshalString function and updated error logging in fig_test.go The UnmarshalString function for the ListenerType has been moved higher up in the fig_test.go code. Updates to error logging formats have also been made for better readability, while unnecessary attributes in the server configuration have been removed. --- fig_test.go | 48 +++++++++++++++++------------------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/fig_test.go b/fig_test.go index 92ea75f..36e0573 100644 --- a/fig_test.go +++ b/fig_test.go @@ -92,6 +92,20 @@ const ( ListenerTLS ) +func (l *ListenerType) UnmarshalString(v string) error { + switch strings.ToLower(v) { + case "unix": + *l = ListenerUnix + case "tcp": + *l = ListenerTCP + case "tls": + *l = ListenerTLS + default: + return fmt.Errorf("unknown listener type: %s", v) + } + return nil +} + func validPodConfig() Pod { var pod Pod @@ -268,7 +282,7 @@ func Test_fig_Load_Defaults(t *testing.T) { want.Logger.Production = false want.Logger.Metadata.Keys = []string{"ts"} want.Application.BuildDate = time.Date(2020, 1, 1, 12, 0, 0, 0, time.UTC) - want.Listener = ListenerTCP + want.Listener = ListenerUnix var cfg Server err := Load(&cfg, File(f), Dirs(filepath.Join("testdata", "valid"))) @@ -375,8 +389,7 @@ func Test_fig_Load_UseStrict(t *testing.T) { for _, f := range []string{"server.yaml", "server.json", "server.toml"} { t.Run(f, func(t *testing.T) { type Server struct { - Host string `fig:"host"` - Listener ListenerType `fig:"listener_type"` + Host string `fig:"host"` } var cfg Server @@ -638,7 +651,7 @@ func Test_fig_decodeMap(t *testing.T) { } if cfg.Server.Listener != ListenerTLS { - t.Errorf("cfg.Server.Listener: want: %s, got: %s", ListenerTLS, cfg.Server.Listener) + t.Errorf("cfg.Server.Listener: want: %d, got: %d", ListenerTLS, cfg.Server.Listener) } } @@ -1280,30 +1293,3 @@ func setenv(t *testing.T, key, value string) { t.Helper() t.Setenv(key, value) } - -func (l *ListenerType) UnmarshalString(v string) error { - switch strings.ToLower(v) { - case "unix": - *l = ListenerUnix - case "tcp": - *l = ListenerTCP - case "tls": - *l = ListenerTLS - default: - return fmt.Errorf("unknown listener type: %s", v) - } - return nil -} - -func (l ListenerType) String() string { - switch l { - case ListenerUnix: - return "unix" - case ListenerTCP: - return "tcp" - case ListenerTLS: - return "tls" - default: - return "unknown" - } -} From a63388017be1d10b00658035ef4ededfd69de507 Mon Sep 17 00:00:00 2001 From: Winni Neessen Date: Tue, 12 Dec 2023 15:39:49 +0100 Subject: [PATCH 11/11] Update UnmarshalString function and enhance error logging in fig.go The UnmarshalString function in fig.go was repositioned for efficiency. An error message for unexpected issues during string unmarshalling was also added. --- fig.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fig.go b/fig.go index 7c60395..3da9d60 100644 --- a/fig.go +++ b/fig.go @@ -337,8 +337,9 @@ func (f *fig) setValue(fv reflect.Value, val string) error { return fmt.Errorf("could not unmarshal string %q: %w", val, err) } fv.Set(reflect.ValueOf(vi).Elem()) + return nil } - return nil + return fmt.Errorf("unexpected error while trying to unmarshal string") } switch fv.Kind() {