Merge pull request miracl#27 from miracl/file-extension

use file extensions to unmarshal data appropriately

.travis.yml

@@ -16,7 +16,7 @@ script:
   - test -z "$(gofmt -s -l . 2>&1 | grep -v vendor | tee /dev/stderr)"
   - golint -set_exit_status ./...
   - ineffassign ./
-  - megacheck ./
+  - megacheck -simple.exit-non-zero -unused.exit-non-zero ./
   - go test -coverprofile .coverprofile
   - $GOPATH/bin/goveralls -v -coverprofile .coverprofile -service=travis-ci
 

conflate.go

@@ -70,12 +70,12 @@ func (c *Conflate) AddURLs(urls ...url.URL) error {
 	if err != nil {
 		return err
 	}
-	return c.MergeData(data...)
+	return c.mergeData(data...)
 }
 
 // AddGo recursively merges the given (json-serializable) golang objects into the Conflate instance
 func (c *Conflate) AddGo(objs ...interface{}) error {
-	data, err := marshalAll(jsonMarshal, objs...)
+	data, err := jsonMarshalAll(objs...)
 	if err != nil {
 		return err
 	}
@@ -84,32 +84,11 @@ func (c *Conflate) AddGo(objs ...interface{}) error {
 
 // AddData recursively merges the given data into the Conflate instance
 func (c *Conflate) AddData(data ...[]byte) error {
-	data, err := loadDataRecursive(nil, data...)
+	fdata, err := wrapFiledatas(data...)
 	if err != nil {
 		return err
 	}
-	return c.MergeData(data...)
-}
-
-// MergeData merges the given data into the Conflate instance
-func (c *Conflate) MergeData(data ...[]byte) error {
-	doms, err := unmarshalAll(unmarshalAny, data...)
-	if err != nil {
-		return err
-	}
-	err = mergeTo(&c.data, doms...)
-	if err != nil {
-		return err
-	}
-	c.removeIncludes()
-	return nil
-}
-
-func (c *Conflate) removeIncludes() {
-	m, ok := c.data.(map[string]interface{})
-	if ok {
-		delete(m, "includes")
-	}
+	return c.addData(fdata...)
 }
 
 // SetSchemaFile loads a JSON v4 schema from the given path
@@ -133,7 +112,7 @@ func (c *Conflate) SetSchemaURL(url url.URL) error {
 // SetSchemaData loads a JSON v4 schema from the given data
 func (c *Conflate) SetSchemaData(data []byte) error {
 	var schema interface{}
-	err := jsonUnmarshal(data, &schema)
+	err := JSONUnmarshal(data, &schema)
 	if err != nil {
 		return wrapError(err, "Schema is not valid json")
 	}
@@ -187,3 +166,16 @@ func (c *Conflate) MarshalTOML() ([]byte, error) {
 func (c *Conflate) MarshalSchema() ([]byte, error) {
 	return jsonMarshal(c.schema)
 }
+
+func (c *Conflate) addData(fdata ...filedata) error {
+	fdata, err := loadDataRecursive(nil, fdata...)
+	if err != nil {
+		return err
+	}
+	return c.mergeData(fdata...)
+}
+
+func (c *Conflate) mergeData(fdata ...filedata) error {
+	doms := filedatas(fdata).objs()
+	return mergeTo(&c.data, doms...)
+}

conflate_test.go

@@ -333,9 +333,16 @@ func TestConflate_MarshalSchemaError(t *testing.T) {
 	assert.Contains(t, err.Error(), "The data could not be marshalled")
 }
 
-func TestConflate_MergeDataError(t *testing.T) {
+func TestConflate_addDataError(t *testing.T) {
 	c := New()
-	err := c.MergeData([]byte("{not valid"))
+	err := c.AddData([]byte(`{"includes": ["missing"]}`))
 	assert.NotNil(t, err)
-	assert.Contains(t, err.Error(), "Could not unmarshal data")
+	assert.Contains(t, err.Error(), "Failed to load url")
+}
+
+func TestConflate_mergeDataError(t *testing.T) {
+	c := New()
+	err := c.AddData([]byte(`"x": {}`), []byte(`"x": []`))
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "Failed to merge")
 }

example/main.go

@@ -7,6 +7,25 @@ import (
 	"runtime"
 )
 
+func init() {
+	// define the unmarshallers for the given file extensions, blank extension is the global unmarshaller
+	conflate.Unmarshallers = conflate.UnmarshallerMap{
+		".json": {customJSONUnmarshal},
+		".jsn":  {conflate.JSONUnmarshal},
+		".yaml": {conflate.YAMLUnmarshal},
+		".yml":  {conflate.YAMLUnmarshal},
+		".toml": {conflate.TOMLUnmarshal},
+		".tml":  {conflate.TOMLUnmarshal},
+		"":      {conflate.JSONUnmarshal, conflate.YAMLUnmarshal, conflate.TOMLUnmarshal},
+	}
+}
+
+// example of a custom unmarshaller for JSON
+func customJSONUnmarshal(data []byte, out interface{}) error {
+	fmt.Println("Using custom JSON Unmarshaller")
+	return conflate.JSONUnmarshal(data, out)
+}
+
 func main() {
 	_, thisFile, _, _ := runtime.Caller(0)
 	thisDir := path.Dir(thisFile)

filedata.go

@@ -0,0 +1,96 @@
+package conflate
+
+import (
+	pkgurl "net/url"
+	"path/filepath"
+	"strings"
+)
+
+type filedata struct {
+	url      pkgurl.URL
+	bytes    []byte
+	obj      map[string]interface{}
+	includes []string
+}
+
+type filedatas []filedata
+
+// UnmarshallerFunc defines the type of function used for unmarshalling data
+type UnmarshallerFunc func([]byte, interface{}) error
+
+// UnmarshallerFuncs defines the type for a slice of UnmarshallerFunc
+type UnmarshallerFuncs []UnmarshallerFunc
+
+// UnmarshallerMap defines the type of a map of string to UnmarshallerFuncs
+type UnmarshallerMap map[string]UnmarshallerFuncs
+
+// Unmarshallers is a list of unmarshalling functions to be used for given file extensions. The unmarshaller slice for the blank file extension is used when no match is found.
+var Unmarshallers = UnmarshallerMap{
+	".json": {JSONUnmarshal},
+	".jsn":  {JSONUnmarshal},
+	".yaml": {YAMLUnmarshal},
+	".yml":  {YAMLUnmarshal},
+	".toml": {TOMLUnmarshal},
+	".tml":  {TOMLUnmarshal},
+	"":      {JSONUnmarshal, YAMLUnmarshal, TOMLUnmarshal},
+}
+
+func newFiledata(bytes []byte, url pkgurl.URL) (filedata, error) {
+	fd := filedata{bytes: bytes, url: url}
+	err := fd.unmarshal()
+	if err != nil {
+		return fd, err
+	}
+	err = fd.extractIncludes()
+	return fd, err
+}
+
+func wrapFiledata(bytes []byte) (filedata, error) {
+	return newFiledata(bytes, emptyURL)
+}
+
+func wrapFiledatas(bytes ...[]byte) (filedatas, error) {
+	var fds []filedata
+	for _, b := range bytes {
+		fd, err := wrapFiledata(b)
+		if err != nil {
+			return nil, err
+		}
+		fds = append(fds, fd)
+	}
+	return fds, nil
+}
+
+func (fd *filedata) unmarshal() error {
+	ext := strings.ToLower(filepath.Ext(fd.url.Path))
+	unmarshallers, ok := Unmarshallers[ext]
+	if !ok {
+		unmarshallers = Unmarshallers[""]
+	}
+	err := makeError("Could not unmarshal data")
+	for _, unmarshal := range unmarshallers {
+		uerr := unmarshal(fd.bytes, &fd.obj)
+		if uerr == nil {
+			return nil
+		}
+		err = wrapError(uerr, err.Error())
+	}
+	return err
+}
+
+func (fd *filedata) extractIncludes() error {
+	err := jsonMarshalUnmarshal(fd.obj["includes"], &fd.includes)
+	if err != nil {
+		return wrapError(err, "Could not extract includes")
+	}
+	delete(fd.obj, "includes")
+	return nil
+}
+
+func (fds filedatas) objs() []interface{} {
+	var objs []interface{}
+	for _, fd := range fds {
+		objs = append(objs, fd.obj)
+	}
+	return objs
+}

filedata_test.go

@@ -0,0 +1,153 @@
+package conflate
+
+import (
+	"github.com/stretchr/testify/assert"
+	pkgurl "net/url"
+	"testing"
+)
+
+func testFiledataNew(t *testing.T, data []byte, path string) (filedata, error) {
+	url, err := pkgurl.Parse(path)
+	assert.Nil(t, err)
+	return newFiledata(data, *url)
+}
+
+func testFiledataNewAssert(t *testing.T, data []byte, path string) filedata {
+	fd, err := testFiledataNew(t, data, path)
+	assert.Nil(t, err)
+	return fd
+}
+
+func TestFiledata_JSONAsAny(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalJSON, "file")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_JSONAsUnknown(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalJSON, "file.unknown")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_JSONAsJSON(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalJSON, "file.json")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_JSONAsJSN(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalJSON, "file.jsn")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_JSONAsTOML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalJSON, "file.toml")
+	assert.NotNil(t, err)
+	assert.Nil(t, fd.obj)
+}
+
+func TestFiledata_YAMLAsAny(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalYAML, "file")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_YAMLAsUnknown(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalYAML, "file.unknown")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_YAMLAsYAML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalYAML, "file.yaml")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_YAMLAsYML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalYAML, "file.yml")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_YAMLAsTOML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalYAML, "file.toml")
+	assert.NotNil(t, err)
+	assert.Nil(t, fd.obj)
+}
+
+func TestFiledata_TOMLAsAny(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalTOML, "file")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_TOMLAsUnknown(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalTOML, "file.unknown")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_TOMLAsTOML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalTOML, "file.toml")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_TOMLAsTML(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalTOML, "file.tml")
+	assert.Nil(t, err)
+	assert.Equal(t, fd.obj, testMarshalData)
+}
+
+func TestFiledata_TOMLAsJSON(t *testing.T) {
+	fd, err := testFiledataNew(t, testMarshalTOML, "file.json")
+	assert.NotNil(t, err)
+	assert.Nil(t, fd.obj)
+}
+
+func TestFiledata_NoIncludes(t *testing.T) {
+	fd, err := wrapFiledata([]byte(`{"x": 1}`))
+	assert.Nil(t, err)
+	assert.Nil(t, fd.obj["includes"])
+	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
+}
+
+func TestFiledata_BlankIncludes(t *testing.T) {
+	fd, err := wrapFiledata([]byte(`{"includes":[], "x": 1}`))
+	assert.Nil(t, err)
+	assert.Nil(t, fd.obj["includes"])
+	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
+}
+
+func TestFiledata_NullIncludes(t *testing.T) {
+	fd, err := wrapFiledata([]byte(`{"includes":null, "x": 1}`))
+	assert.Nil(t, err)
+	assert.Nil(t, fd.obj["includes"])
+	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
+}
+
+func TestFiledata_Includes(t *testing.T) {
+	fd, err := wrapFiledata([]byte(`{"includes":["test1", "test2"], "x": 1}`))
+	assert.Nil(t, err)
+	assert.Equal(t, fd.includes, []string{"test1", "test2"})
+	assert.Nil(t, fd.obj["includes"])
+	assert.Equal(t, fd.obj, map[string]interface{}{"x": 1.0})
+}
+
+func TestFiledata_IncludesError(t *testing.T) {
+	_, err := wrapFiledata([]byte(`{"includes": "not array"}`))
+	assert.NotNil(t, err)
+	assert.Contains(t, err.Error(), "Could not extract includes")
+}
+
+func TestFiledatas_Unmarshal(t *testing.T) {
+	fds := filedatas{
+		testFiledataNewAssert(t, testMarshalJSON, "file.json"),
+		testFiledataNewAssert(t, testMarshalYAML, "file.yaml"),
+		testFiledataNewAssert(t, testMarshalTOML, "file.toml"),
+	}
+	assert.Equal(t, fds.objs(), []interface{}{testMarshalData, testMarshalData, testMarshalData})
+}