Add support for creating repository indexes in JSON format (#739)

* repo: add support for JSON marshalling Signed-off-by: MeurillonGuillaume <[email protected]> * cfg: add configuration for json index Signed-off-by: MeurillonGuillaume <[email protected]> * repo: add tests for json and yaml index options Signed-off-by: MeurillonGuillaume <[email protected]> --------- Signed-off-by: MeurillonGuillaume <[email protected]> Co-authored-by: MeurillonGuillaume <[email protected]>
helm · Jan 29, 2024 · 90e47f6 · 90e47f6
1 parent f00683f
commit 90e47f6
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 23 deletions.
diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go
@@ -119,6 +119,7 @@ func cliHandler(c *cli.Context) {
 		WebTemplatePath:        conf.GetString("web-template-path"),
 		ArtifactHubRepoID:      conf.GetStringMapString("artifact-hub-repo-id"),
 		AlwaysRegenerateIndex:  conf.GetBool("always-regenerate-chart-index"),
+		JSONIndex:              conf.GetBool("json-index"),
 	}
 
 	server, err := newServer(options)

diff --git a/pkg/chartmuseum/server.go b/pkg/chartmuseum/server.go
@@ -84,6 +84,7 @@ type (
 		// AlwaysRegenerateIndex represents if the museum always return the up-to-date chart
 		// which means that the GetChart will increase its latency , be careful to enable this .
 		AlwaysRegenerateIndex bool
+		JSONIndex             bool
 	}
 
 	// Server is a generic interface for web servers
@@ -151,6 +152,7 @@ func NewServer(options ServerOptions) (Server, error) {
 		// EnforceSemver2 - see https://github.com/helm/chartmuseum/issues/485 for more info
 		EnforceSemver2:        options.EnforceSemver2,
 		AlwaysRegenerateIndex: options.AlwaysRegenerateIndex,
+		JSONIndex:             options.JSONIndex,
 	})
 
 	return server, err

diff --git a/pkg/chartmuseum/server/multitenant/cache.go b/pkg/chartmuseum/server/multitenant/cache.go
@@ -152,11 +152,12 @@ func (server *MultiTenantServer) regenerateRepositoryIndexWorker(log cm_logger.L
 		"repo", repo,
 	)
 	index := &cm_repo.Index{
-		IndexFile: entry.RepoIndex.IndexFile,
-		RepoName:  repo,
-		Raw:       entry.RepoIndex.Raw,
-		ChartURL:  entry.RepoIndex.ChartURL,
-		IndexLock: sync.RWMutex{},
+		IndexFile:  entry.RepoIndex.IndexFile,
+		RepoName:   repo,
+		Raw:        entry.RepoIndex.Raw,
+		ChartURL:   entry.RepoIndex.ChartURL,
+		IndexLock:  sync.RWMutex{},
+		OutputJSON: server.JSONIndex,
 	}
 
 	for _, object := range diff.Removed {
@@ -442,35 +443,41 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep
 	}
 
 	if !server.UseStatefiles {
-		return cm_repo.NewIndex(chartURL, repo, serverInfo)
+		return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
 	}
 
 	objectPath := pathutil.Join(repo, cm_repo.StatefileFilename)
 	object, err := server.StorageBackend.GetObject(objectPath)
 	if err != nil {
-		return cm_repo.NewIndex(chartURL, repo, serverInfo)
+		return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
 	}
 
 	indexFile := &cm_repo.IndexFile{}
-	err = yaml.Unmarshal(object.Content, indexFile)
+	if json.Valid(object.Content) {
+		err = json.Unmarshal(object.Content, indexFile)
+	} else {
+		err = yaml.Unmarshal(object.Content, indexFile)
+	}
+
 	if err != nil {
 		log(cm_logger.WarnLevel, "index-cache.yaml found but could not be parsed",
 			"repo", repo,
 			"error", err.Error(),
 		)
-		return cm_repo.NewIndex(chartURL, repo, serverInfo)
+		return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
 	}
 
 	log(cm_logger.DebugLevel, "index-cache.yaml loaded",
 		"repo", repo,
 	)
 
 	return &cm_repo.Index{
-		IndexFile: indexFile,
-		RepoName:  repo,
-		Raw:       object.Content,
-		ChartURL:  chartURL,
-		IndexLock: sync.RWMutex{},
+		IndexFile:  indexFile,
+		RepoName:   repo,
+		Raw:        object.Content,
+		ChartURL:   chartURL,
+		IndexLock:  sync.RWMutex{},
+		OutputJSON: server.JSONIndex,
 	}
 }
 

diff --git a/pkg/chartmuseum/server/multitenant/server.go b/pkg/chartmuseum/server/multitenant/server.go
@@ -73,6 +73,7 @@ type (
 		EnforceSemver2        bool
 		WebTemplatePath       string
 		AlwaysRegenerateIndex bool
+		JSONIndex             bool
 	}
 
 	ObjectsPerChartLimit struct {
@@ -106,6 +107,7 @@ type (
 		// Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info
 		EnforceSemver2        bool
 		AlwaysRegenerateIndex bool
+		JSONIndex             bool
 	}
 
 	tenantInternals struct {
@@ -166,6 +168,7 @@ func NewMultiTenantServer(options MultiTenantServerOptions) (*MultiTenantServer,
 		WebTemplatePath:        options.WebTemplatePath,
 		ArtifactHubRepoID:      options.ArtifactHubRepoID,
 		AlwaysRegenerateIndex:  options.AlwaysRegenerateIndex,
+		JSONIndex:              options.JSONIndex,
 	}
 
 	if server.WebTemplatePath != "" {

diff --git a/pkg/config/vars.go b/pkg/config/vars.go
@@ -138,6 +138,15 @@ var configVars = map[string]configVar{
 			EnvVar: "DISABLE_STATEFILES",
 		},
 	},
+	"json-index": {
+		Type:    boolType,
+		Default: false,
+		CLIFlag: cli.BoolFlag{
+			Name:   "json-index",
+			Usage:  "generates an index in JSON format, improves parsing performance for large index files",
+			EnvVar: "JSON_INDEX",
+		},
+	},
 	"allowoverwrite": {
 		Type:    boolType,
 		Default: false,

diff --git a/pkg/repo/index.go b/pkg/repo/index.go
@@ -17,6 +17,7 @@ limitations under the License.
 package repo
 
 import (
+	"encoding/json"
 	"sync"
 	"time"
 
@@ -51,27 +52,34 @@ type (
 		Raw        []byte `json:"c"`
 		ChartURL   string `json:"d"`
 		IndexLock  sync.RWMutex
+		OutputJSON bool
 	}
 )
 
 // NewIndex creates a new instance of Index
-func NewIndex(chartURL string, repo string, serverInfo *ServerInfo) *Index {
+func NewIndex(chartURL string, repo string, serverInfo *ServerInfo, outputJSON bool) *Index {
 	indexFile := &IndexFile{
 		IndexFile:  &helm_repo.IndexFile{},
 		ServerInfo: serverInfo,
 	}
-	index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}}
+	index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}, outputJSON}
 	index.Entries = map[string]helm_repo.ChartVersions{}
 	index.APIVersion = helm_repo.APIVersionV1
 	index.Regenerate()
 	return &index
 }
 
 // Regenerate sorts entries in index file and sets current time for generated key
-func (index *Index) Regenerate() error {
+func (index *Index) Regenerate() (err error) {
 	index.SortEntries()
 	index.Generated = time.Now().Round(time.Second)
-	raw, err := yaml.Marshal(index.IndexFile)
+
+	var raw []byte
+	if index.OutputJSON {
+		raw, err = json.Marshal(index.IndexFile)
+	} else {
+		raw, err = yaml.Marshal(index.IndexFile)
+	}
 	if err != nil {
 		return err
 	}

diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go
@@ -17,13 +17,15 @@ limitations under the License.
 package repo
 
 import (
+	"encoding/json"
 	"fmt"
 	"testing"
 	"time"
 
 	"strings"
 
 	"github.com/stretchr/testify/suite"
+	"sigs.k8s.io/yaml"
 
 	"helm.sh/helm/v3/pkg/chart"
 	helm_repo "helm.sh/helm/v3/pkg/repo"
@@ -51,7 +53,7 @@ func getChartVersion(name string, patch int, created time.Time) *helm_repo.Chart
 }
 
 func (suite *IndexTestSuite) SetupSuite() {
-	suite.Index = NewIndex("", "", &ServerInfo{})
+	suite.Index = NewIndex("", "", &ServerInfo{}, false)
 	now := time.Now()
 	for _, name := range []string{"a", "b", "c"} {
 		for i := 0; i < 10; i++ {
@@ -94,13 +96,13 @@ func (suite *IndexTestSuite) TestRemove() {
 }
 
 func (suite *IndexTestSuite) TestChartURLs() {
-	index := NewIndex("", "", &ServerInfo{})
+	index := NewIndex("", "", &ServerInfo{}, false)
 	chartVersion := getChartVersion("a", 0, time.Now())
 	index.AddEntry(chartVersion)
 	suite.Equal("charts/a-1.0.0.tgz",
 		index.Entries["a"][0].URLs[0], "relative chart url")
 
-	index = NewIndex("http://mysite.com:8080", "", &ServerInfo{})
+	index = NewIndex("http://mysite.com:8080", "", &ServerInfo{}, false)
 	chartVersion = getChartVersion("a", 0, time.Now())
 	index.AddEntry(chartVersion)
 	suite.Equal("http://mysite.com:8080/charts/a-1.0.0.tgz",
@@ -109,16 +111,37 @@ func (suite *IndexTestSuite) TestChartURLs() {
 
 func (suite *IndexTestSuite) TestServerInfo() {
 	serverInfo := &ServerInfo{}
-	index := NewIndex("", "", serverInfo)
+	index := NewIndex("", "", serverInfo, false)
 	suite.False(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path not in index")
 
 	serverInfo = &ServerInfo{
 		ContextPath: "/v1/helm",
 	}
-	index = NewIndex("", "", serverInfo)
+	index = NewIndex("", "", serverInfo, false)
 	suite.True(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path is in index")
 }
 
+func (suite *IndexTestSuite) TestYAMLIndex() {
+	index := NewIndex("", "", &ServerInfo{}, false)
+	chartVersion := getChartVersion("a", 0, time.Now())
+	index.AddEntry(chartVersion)
+	suite.NoError(index.Regenerate())
+
+	suite.False(json.Valid(index.Raw))
+	suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{}))
+}
+
+func (suite *IndexTestSuite) TestJSONIndex() {
+	index := NewIndex("", "", &ServerInfo{}, true)
+	chartVersion := getChartVersion("a", 0, time.Now())
+	index.AddEntry(chartVersion)
+	suite.NoError(index.Regenerate())
+
+	// Since YAML is a superset of JSON, any valid JSON should be valid YAML
+	suite.True(json.Valid(index.Raw))
+	suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{}))
+}
+
 func TestIndexTestSuite(t *testing.T) {
 	suite.Run(t, new(IndexTestSuite))
 }