diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go index 86089fa6..77f7ad49 100644 --- a/cmd/chartmuseum/main.go +++ b/cmd/chartmuseum/main.go @@ -119,6 +119,7 @@ func cliHandler(c *cli.Context) { WebTemplatePath: conf.GetString("web-template-path"), ArtifactHubRepoID: conf.GetStringMapString("artifact-hub-repo-id"), AlwaysRegenerateIndex: conf.GetBool("always-regenerate-chart-index"), + JSONIndex: conf.GetBool("json-index"), } server, err := newServer(options) diff --git a/pkg/chartmuseum/server.go b/pkg/chartmuseum/server.go index 568dfd36..c9d88f03 100644 --- a/pkg/chartmuseum/server.go +++ b/pkg/chartmuseum/server.go @@ -84,6 +84,7 @@ type ( // AlwaysRegenerateIndex represents if the museum always return the up-to-date chart // which means that the GetChart will increase its latency , be careful to enable this . AlwaysRegenerateIndex bool + JSONIndex bool } // Server is a generic interface for web servers @@ -151,6 +152,7 @@ func NewServer(options ServerOptions) (Server, error) { // EnforceSemver2 - see https://github.com/helm/chartmuseum/issues/485 for more info EnforceSemver2: options.EnforceSemver2, AlwaysRegenerateIndex: options.AlwaysRegenerateIndex, + JSONIndex: options.JSONIndex, }) return server, err diff --git a/pkg/chartmuseum/server/multitenant/cache.go b/pkg/chartmuseum/server/multitenant/cache.go index e7cc69e4..60fdfb9b 100644 --- a/pkg/chartmuseum/server/multitenant/cache.go +++ b/pkg/chartmuseum/server/multitenant/cache.go @@ -152,11 +152,12 @@ func (server *MultiTenantServer) regenerateRepositoryIndexWorker(log cm_logger.L "repo", repo, ) index := &cm_repo.Index{ - IndexFile: entry.RepoIndex.IndexFile, - RepoName: repo, - Raw: entry.RepoIndex.Raw, - ChartURL: entry.RepoIndex.ChartURL, - IndexLock: sync.RWMutex{}, + IndexFile: entry.RepoIndex.IndexFile, + RepoName: repo, + Raw: entry.RepoIndex.Raw, + ChartURL: entry.RepoIndex.ChartURL, + IndexLock: sync.RWMutex{}, + OutputJSON: server.JSONIndex, } for _, object := range diff.Removed { @@ -442,23 +443,28 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep } if !server.UseStatefiles { - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } objectPath := pathutil.Join(repo, cm_repo.StatefileFilename) object, err := server.StorageBackend.GetObject(objectPath) if err != nil { - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } indexFile := &cm_repo.IndexFile{} - err = yaml.Unmarshal(object.Content, indexFile) + if json.Valid(object.Content) { + err = json.Unmarshal(object.Content, indexFile) + } else { + err = yaml.Unmarshal(object.Content, indexFile) + } + if err != nil { log(cm_logger.WarnLevel, "index-cache.yaml found but could not be parsed", "repo", repo, "error", err.Error(), ) - return cm_repo.NewIndex(chartURL, repo, serverInfo) + return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex) } log(cm_logger.DebugLevel, "index-cache.yaml loaded", @@ -466,11 +472,12 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep ) return &cm_repo.Index{ - IndexFile: indexFile, - RepoName: repo, - Raw: object.Content, - ChartURL: chartURL, - IndexLock: sync.RWMutex{}, + IndexFile: indexFile, + RepoName: repo, + Raw: object.Content, + ChartURL: chartURL, + IndexLock: sync.RWMutex{}, + OutputJSON: server.JSONIndex, } } diff --git a/pkg/chartmuseum/server/multitenant/server.go b/pkg/chartmuseum/server/multitenant/server.go index d6221f33..bb5c7437 100644 --- a/pkg/chartmuseum/server/multitenant/server.go +++ b/pkg/chartmuseum/server/multitenant/server.go @@ -73,6 +73,7 @@ type ( EnforceSemver2 bool WebTemplatePath string AlwaysRegenerateIndex bool + JSONIndex bool } ObjectsPerChartLimit struct { @@ -106,6 +107,7 @@ type ( // Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info EnforceSemver2 bool AlwaysRegenerateIndex bool + JSONIndex bool } tenantInternals struct { @@ -166,6 +168,7 @@ func NewMultiTenantServer(options MultiTenantServerOptions) (*MultiTenantServer, WebTemplatePath: options.WebTemplatePath, ArtifactHubRepoID: options.ArtifactHubRepoID, AlwaysRegenerateIndex: options.AlwaysRegenerateIndex, + JSONIndex: options.JSONIndex, } if server.WebTemplatePath != "" { diff --git a/pkg/config/vars.go b/pkg/config/vars.go index e3cfd0c1..668b2637 100644 --- a/pkg/config/vars.go +++ b/pkg/config/vars.go @@ -138,6 +138,15 @@ var configVars = map[string]configVar{ EnvVar: "DISABLE_STATEFILES", }, }, + "json-index": { + Type: boolType, + Default: false, + CLIFlag: cli.BoolFlag{ + Name: "json-index", + Usage: "generates an index in JSON format, improves parsing performance for large index files", + EnvVar: "JSON_INDEX", + }, + }, "allowoverwrite": { Type: boolType, Default: false, diff --git a/pkg/repo/index.go b/pkg/repo/index.go index c588a6db..1f68d5b7 100644 --- a/pkg/repo/index.go +++ b/pkg/repo/index.go @@ -17,6 +17,7 @@ limitations under the License. package repo import ( + "encoding/json" "sync" "time" @@ -51,16 +52,17 @@ type ( Raw []byte `json:"c"` ChartURL string `json:"d"` IndexLock sync.RWMutex + OutputJSON bool } ) // NewIndex creates a new instance of Index -func NewIndex(chartURL string, repo string, serverInfo *ServerInfo) *Index { +func NewIndex(chartURL string, repo string, serverInfo *ServerInfo, outputJSON bool) *Index { indexFile := &IndexFile{ IndexFile: &helm_repo.IndexFile{}, ServerInfo: serverInfo, } - index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}} + index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}, outputJSON} index.Entries = map[string]helm_repo.ChartVersions{} index.APIVersion = helm_repo.APIVersionV1 index.Regenerate() @@ -68,10 +70,16 @@ func NewIndex(chartURL string, repo string, serverInfo *ServerInfo) *Index { } // Regenerate sorts entries in index file and sets current time for generated key -func (index *Index) Regenerate() error { +func (index *Index) Regenerate() (err error) { index.SortEntries() index.Generated = time.Now().Round(time.Second) - raw, err := yaml.Marshal(index.IndexFile) + + var raw []byte + if index.OutputJSON { + raw, err = json.Marshal(index.IndexFile) + } else { + raw, err = yaml.Marshal(index.IndexFile) + } if err != nil { return err } diff --git a/pkg/repo/index_test.go b/pkg/repo/index_test.go index d60f06ca..af480fdd 100644 --- a/pkg/repo/index_test.go +++ b/pkg/repo/index_test.go @@ -17,6 +17,7 @@ limitations under the License. package repo import ( + "encoding/json" "fmt" "testing" "time" @@ -24,6 +25,7 @@ import ( "strings" "github.com/stretchr/testify/suite" + "sigs.k8s.io/yaml" "helm.sh/helm/v3/pkg/chart" helm_repo "helm.sh/helm/v3/pkg/repo" @@ -51,7 +53,7 @@ func getChartVersion(name string, patch int, created time.Time) *helm_repo.Chart } func (suite *IndexTestSuite) SetupSuite() { - suite.Index = NewIndex("", "", &ServerInfo{}) + suite.Index = NewIndex("", "", &ServerInfo{}, false) now := time.Now() for _, name := range []string{"a", "b", "c"} { for i := 0; i < 10; i++ { @@ -94,13 +96,13 @@ func (suite *IndexTestSuite) TestRemove() { } func (suite *IndexTestSuite) TestChartURLs() { - index := NewIndex("", "", &ServerInfo{}) + index := NewIndex("", "", &ServerInfo{}, false) chartVersion := getChartVersion("a", 0, time.Now()) index.AddEntry(chartVersion) suite.Equal("charts/a-1.0.0.tgz", index.Entries["a"][0].URLs[0], "relative chart url") - index = NewIndex("http://mysite.com:8080", "", &ServerInfo{}) + index = NewIndex("http://mysite.com:8080", "", &ServerInfo{}, false) chartVersion = getChartVersion("a", 0, time.Now()) index.AddEntry(chartVersion) suite.Equal("http://mysite.com:8080/charts/a-1.0.0.tgz", @@ -109,16 +111,37 @@ func (suite *IndexTestSuite) TestChartURLs() { func (suite *IndexTestSuite) TestServerInfo() { serverInfo := &ServerInfo{} - index := NewIndex("", "", serverInfo) + index := NewIndex("", "", serverInfo, false) suite.False(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path not in index") serverInfo = &ServerInfo{ ContextPath: "/v1/helm", } - index = NewIndex("", "", serverInfo) + index = NewIndex("", "", serverInfo, false) suite.True(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path is in index") } +func (suite *IndexTestSuite) TestYAMLIndex() { + index := NewIndex("", "", &ServerInfo{}, false) + chartVersion := getChartVersion("a", 0, time.Now()) + index.AddEntry(chartVersion) + suite.NoError(index.Regenerate()) + + suite.False(json.Valid(index.Raw)) + suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{})) +} + +func (suite *IndexTestSuite) TestJSONIndex() { + index := NewIndex("", "", &ServerInfo{}, true) + chartVersion := getChartVersion("a", 0, time.Now()) + index.AddEntry(chartVersion) + suite.NoError(index.Regenerate()) + + // Since YAML is a superset of JSON, any valid JSON should be valid YAML + suite.True(json.Valid(index.Raw)) + suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{})) +} + func TestIndexTestSuite(t *testing.T) { suite.Run(t, new(IndexTestSuite)) }