diff --git a/pkg/cmd/build/build.go b/pkg/cmd/build/build.go index 4c5529d0..e080a6db 100644 --- a/pkg/cmd/build/build.go +++ b/pkg/cmd/build/build.go @@ -117,12 +117,13 @@ func (options *BuildOptions) RunBuild() error { model.Config = jozufile modelStorePath := options.storageHome + repo := "" tag := "" if options.modelRef != nil { - modelStorePath = path.Join(options.storageHome, options.modelRef.Registry, options.modelRef.Repository) + repo = path.Join(options.modelRef.Registry, options.modelRef.Repository) tag = options.modelRef.Reference } - store := storage.NewLocalStore(modelStorePath) + store := storage.NewLocalStore(modelStorePath, repo) manifestDesc, err := store.SaveModel(model, tag) if err != nil { fmt.Println(err) diff --git a/pkg/cmd/models/cmd.go b/pkg/cmd/models/cmd.go index 5cc3b8d5..492a501d 100644 --- a/pkg/cmd/models/cmd.go +++ b/pkg/cmd/models/cmd.go @@ -2,7 +2,12 @@ package models import ( "fmt" + "io/fs" "jmm/pkg/lib/storage" + "os" + "path" + "path/filepath" + "text/tabwriter" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -18,11 +23,13 @@ var ( ) type ModelsOptions struct { - configHome string + configHome string + storageHome string } func (opts *ModelsOptions) complete() { opts.configHome = viper.GetString("config") + opts.storageHome = path.Join(opts.configHome, "storage") } func (opts *ModelsOptions) validate() error { @@ -51,14 +58,62 @@ func RunCommand(options *ModelsOptions) func(*cobra.Command, []string) { fmt.Println(err) return } - store := storage.NewLocalStore(opts.configHome) - summary, err := listModels(store) + storeDirs, err := findRepos(opts.storageHome) if err != nil { fmt.Println(err) - return } - fmt.Println(summary) + var allInfoLines []string + for _, storeDir := range storeDirs { + store := storage.NewLocalStore(opts.storageHome, storeDir) + + infolines, err := listModels(store) + if err != nil { + fmt.Println(err) + return + } + allInfoLines = append(allInfoLines, infolines...) + } + + printSummary(allInfoLines) + + } +} + +func findRepos(storePath string) ([]string, error) { + var indexPaths []string + err := filepath.WalkDir(storePath, func(file string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + if info.Name() == "index.json" && !info.IsDir() { + dir := filepath.Dir(file) + relDir, err := filepath.Rel(storePath, dir) + if err != nil { + return err + } + if relDir == "." { + relDir = "" + } + indexPaths = append(indexPaths, relDir) + } + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to read local storage: %w", err) + } + return indexPaths, nil +} + +func printSummary(lines []string) { + tw := tabwriter.NewWriter(os.Stdout, 0, 2, 3, ' ', 0) + fmt.Fprintln(tw, ModelsTableHeader) + for _, line := range lines { + fmt.Fprintln(tw, line) } + tw.Flush() } diff --git a/pkg/cmd/models/models.go b/pkg/cmd/models/models.go index 70f22fbe..5f3c72b0 100644 --- a/pkg/cmd/models/models.go +++ b/pkg/cmd/models/models.go @@ -4,60 +4,62 @@ Copyright © 2024 Jozu.com package models import ( - "bytes" "context" "encoding/json" "fmt" "jmm/pkg/artifact" "jmm/pkg/lib/storage" "math" - "text/tabwriter" + "strings" - "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) const ( - ModelsTableHeader = "DIGEST\tMAINTAINER\tMODEL FORMAT\tSIZE" - ModelsTableFmt = "%s\t%s\t%s\t%s\t" + ModelsTableHeader = "REPOSITORY\tTAG\tMAINTAINER\tMODEL FORMAT\tSIZE\tDIGEST" + ModelsTableFmt = "%s\t%s\t%s\t%s\t%s\t%s\t" ) -func listModels(store storage.Store) (string, error) { +func listModels(store storage.Store) ([]string, error) { index, err := store.ParseIndexJson() if err != nil { - return "", err + return nil, err } - manifests, err := manifestsFromIndex(index, store) - if err != nil { - return "", err - } - - summary, err := printManifestsSummary(manifests, store) - if err != nil { - return "", err - } - - return summary, nil -} - -func manifestsFromIndex(index *ocispec.Index, store storage.Store) (map[digest.Digest]ocispec.Manifest, error) { - manifests := map[digest.Digest]ocispec.Manifest{} + var infolines []string for _, manifestDesc := range index.Manifests { - manifestBytes, err := store.Fetch(context.Background(), manifestDesc) + manifest, err := getManifest(store, manifestDesc) if err != nil { - return nil, fmt.Errorf("failed to read manifest %s: %w", manifestDesc.Digest, err) + return nil, err } - manifest := ocispec.Manifest{} - if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, fmt.Errorf("failed to parse manifest %s: %w", manifestDesc.Digest, err) + manifestConf, err := readManifestConfig(store, manifest) + if err != nil { + return nil, err + } + // TODO: filter list for our manifests only, ignore other artifacts + infoline, err := getManifestInfoLine(store.GetRepository(), manifestDesc, manifest, manifestConf) + if err != nil { + return nil, err } - manifests[manifestDesc.Digest] = manifest + infolines = append(infolines, infoline) } - return manifests, nil + + return infolines, nil +} + +func getManifest(store storage.Store, manifestDesc ocispec.Descriptor) (*ocispec.Manifest, error) { + manifestBytes, err := store.Fetch(context.Background(), manifestDesc) + if err != nil { + return nil, fmt.Errorf("failed to read manifest %s: %w", manifestDesc.Digest, err) + } + manifest := &ocispec.Manifest{} + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, fmt.Errorf("failed to parse manifest %s: %w", manifestDesc.Digest, err) + } + return manifest, nil } -func readManifestConfig(manifest *ocispec.Manifest, store storage.Store) (*artifact.JozuFile, error) { +func readManifestConfig(store storage.Store, manifest *ocispec.Manifest) (*artifact.JozuFile, error) { configBytes, err := store.Fetch(context.Background(), manifest.Config) if err != nil { return nil, fmt.Errorf("failed to read config: %w", err) @@ -69,34 +71,25 @@ func readManifestConfig(manifest *ocispec.Manifest, store storage.Store) (*artif return config, nil } -func printManifestsSummary(manifests map[digest.Digest]ocispec.Manifest, store storage.Store) (string, error) { - buf := bytes.Buffer{} - tw := tabwriter.NewWriter(&buf, 0, 2, 3, ' ', 0) - fmt.Fprintln(tw, ModelsTableHeader) - for digest, manifest := range manifests { - // TODO: filter this list for manifests we're interested in (build needs to set a manifest mediaType/artifactType) - line, err := getManifestInfoLine(digest, &manifest, store) - if err != nil { - return "", err - } - fmt.Fprintln(tw, line) +func getManifestInfoLine(repo string, desc ocispec.Descriptor, manifest *ocispec.Manifest, config *artifact.JozuFile) (string, error) { + ref := desc.Annotations[ocispec.AnnotationRefName] + if ref == "" { + ref = "" } - tw.Flush() - return buf.String(), nil -} -func getManifestInfoLine(digest digest.Digest, manifest *ocispec.Manifest, store storage.Store) (string, error) { - config, err := readManifestConfig(manifest, store) - if err != nil { - return "", err + // Strip localhost from repo if present, since we added it + repo = strings.TrimPrefix(repo, "localhost/") + if repo == "" { + repo = "" } + var size int64 for _, layer := range manifest.Layers { size += layer.Size } sizeStr := formatBytes(size) - info := fmt.Sprintf(ModelsTableFmt, digest, config.Maintainer, config.ModelFormat, sizeStr) + info := fmt.Sprintf(ModelsTableFmt, repo, ref, config.Maintainer, config.ModelFormat, sizeStr, desc.Digest) return info, nil } diff --git a/pkg/cmd/models/models_test.go b/pkg/cmd/models/models_test.go index a242d490..2aadaba8 100644 --- a/pkg/cmd/models/models_test.go +++ b/pkg/cmd/models/models_test.go @@ -14,7 +14,8 @@ import ( func TestListModels(t *testing.T) { tests := []struct { - name string + testName string + repo string manifests map[digest.Digest]ocispec.Manifest configs map[digest.Digest]artifact.JozuFile index *ocispec.Index @@ -22,16 +23,16 @@ func TestListModels(t *testing.T) { expectErrRegexp string }{ { - name: "Cannot read index.json", + testName: "Cannot read index.json", index: nil, expectErrRegexp: "artifact not found", }, { - name: "Cannot find manifest from index.json", + testName: "Cannot find manifest from index.json", index: &ocispec.Index{ Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", true), - ManifestDesc("manifestNotFound", true), + ManifestDesc("manifestA", "", true), + ManifestDesc("manifestNotFound", "", true), }, }, manifests: map[digest.Digest]ocispec.Manifest{ @@ -45,11 +46,11 @@ func TestListModels(t *testing.T) { expectErrRegexp: "failed to read manifest manifestNotFound.*", }, { - name: "Cannot find config in manifest", + testName: "Cannot find config in manifest", index: &ocispec.Index{ Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", true), - ManifestDesc("manifestB", true), + ManifestDesc("manifestA", "", true), + ManifestDesc("manifestB", "", true), }, }, manifests: map[digest.Digest]ocispec.Manifest{ @@ -63,11 +64,73 @@ func TestListModels(t *testing.T) { expectErrRegexp: "failed to read config.*", }, { - name: "Prints summary of for each manifest line (layers are 1024 bytes)", + testName: "Catches invalid size error", index: &ocispec.Index{ Manifests: []ocispec.Descriptor{ - ManifestDesc("manifestA", true), - ManifestDesc("manifestB", true), + ManifestDesc("manifestA", "", true), + ManifestDesc("manifestB", "", false), + }, + }, + manifests: map[digest.Digest]ocispec.Manifest{ + "manifestA": Manifest("configA", "layerA"), + "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), + }, + configs: map[digest.Digest]artifact.JozuFile{ + "configA": Config("maintainerA", "formatA"), + "configB": Config("maintainerB", "formatB"), + }, + expectErrRegexp: "failed to read manifest manifestB: invalid size", + }, + { + testName: "Prints summary of for each manifest line (layers are 1024 bytes)", + index: &ocispec.Index{ + Manifests: []ocispec.Descriptor{ + ManifestDesc("manifestA", "", true), + ManifestDesc("manifestB", "", true), + }, + }, + manifests: map[digest.Digest]ocispec.Manifest{ + "manifestA": Manifest("configA", "layerA"), + "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), + }, + configs: map[digest.Digest]artifact.JozuFile{ + "configA": Config("maintainerA", "formatA"), + "configB": Config("maintainerB", "formatB"), + }, + expectedOutputRegexps: []string{ + "\t\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", + "\t\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", + }, + }, + { + testName: "Prints summary of for each manifest line including repo and tag", + repo: "testregistry/testrepo", + index: &ocispec.Index{ + Manifests: []ocispec.Descriptor{ + ManifestDesc("manifestA", "tagA", true), + ManifestDesc("manifestB", "tagB", true), + }, + }, + manifests: map[digest.Digest]ocispec.Manifest{ + "manifestA": Manifest("configA", "layerA"), + "manifestB": Manifest("configB", "layerB1", "layerB2", "layerB3"), + }, + configs: map[digest.Digest]artifact.JozuFile{ + "configA": Config("maintainerA", "formatA"), + "configB": Config("maintainerB", "formatB"), + }, + expectedOutputRegexps: []string{ + "testregistry/testrepo\ttagA\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", + "testregistry/testrepo\ttagB\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", + }, + }, + { + testName: "Prints summary of for each manifest line, stripping localhost/ if present", + repo: "localhost/testrepo", + index: &ocispec.Index{ + Manifests: []ocispec.Descriptor{ + ManifestDesc("manifestA", "tagA", true), + ManifestDesc("manifestB", "", true), }, }, manifests: map[digest.Digest]ocispec.Manifest{ @@ -79,23 +142,24 @@ func TestListModels(t *testing.T) { "configB": Config("maintainerB", "formatB"), }, expectedOutputRegexps: []string{ - "manifestA.*maintainerA.*formatA.*1.0 KiB", - "manifestB.*maintainerB.*formatB.*3.0 KiB", + "testrepo\ttagA\tmaintainerA\tformatA\t1.0 KiB\tmanifestA\t", + "testrepo\t\tmaintainerB\tformatB\t3.0 KiB\tmanifestB\t", }, }, } for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + t.Run(tt.testName, func(t *testing.T) { testStore := &internal.TestStore{ Manifests: tt.manifests, Configs: tt.configs, Index: tt.index, + Repo: tt.repo, } - summary, err := listModels(testStore) + summaryLines, err := listModels(testStore) if tt.expectErrRegexp != "" { // Should be error - assert.Empty(t, summary, "Should not output summary on error") - if assert.Error(t, err, "Should return an error") { + assert.Empty(t, summaryLines, "Should not output summary on error") + if !assert.Error(t, err, "Should return an error") { return } assert.Regexp(t, tt.expectErrRegexp, err.Error()) @@ -105,7 +169,7 @@ func TestListModels(t *testing.T) { } for _, line := range tt.expectedOutputRegexps { // Assert all lines in expected output are somewhere in the summary - assert.Regexp(t, line, summary) + assert.Contains(t, summaryLines, line) } } }) @@ -167,15 +231,19 @@ func Config(maintainer, format string) artifact.JozuFile { return config } -func ManifestDesc(digestStr string, valid bool) ocispec.Descriptor { +func ManifestDesc(digestStr string, tag string, valid bool) ocispec.Descriptor { size := internal.ValidSize if !valid { size = internal.InvalidSize } - - return ocispec.Descriptor{ - Digest: digest.Digest(digestStr), - MediaType: ocispec.MediaTypeImageManifest, - Size: size, + desc := ocispec.Descriptor{ + Digest: digest.Digest(digestStr), + MediaType: ocispec.MediaTypeImageManifest, + Size: size, + Annotations: map[string]string{}, + } + if tag != "" { + desc.Annotations[ocispec.AnnotationRefName] = tag } + return desc } diff --git a/pkg/lib/storage/local.go b/pkg/lib/storage/local.go index 7e11e115..5a55fefa 100644 --- a/pkg/lib/storage/local.go +++ b/pkg/lib/storage/local.go @@ -20,12 +20,14 @@ import ( type LocalStore struct { storage *oci.Store indexPath string + repo string } // Assert LocalStore implements the Store interface. var _ Store = (*LocalStore)(nil) -func NewLocalStore(storeHome string) Store { +func NewLocalStore(storeRoot, repo string) Store { + storeHome := filepath.Join(storeRoot, repo) indexPath := filepath.Join(storeHome, "index.json") store, err := oci.New(storeHome) @@ -36,6 +38,7 @@ func NewLocalStore(storeHome string) Store { return &LocalStore{ storage: store, indexPath: indexPath, + repo: repo, } } @@ -44,11 +47,12 @@ func (store *LocalStore) SaveModel(model *artifact.Model, tag string) (*ocispec. if err != nil { return nil, err } - for _, layer := range model.Layers { - _, err = store.saveContentLayer(&layer) + for idx, layer := range model.Layers { + layerDesc, err := store.saveContentLayer(&layer) if err != nil { return nil, err } + model.Layers[idx].Descriptor = *layerDesc } manifestDesc, err := store.saveModelManifest(model, config, tag) @@ -89,6 +93,10 @@ func (store *LocalStore) ParseIndexJson() (*ocispec.Index, error) { return index, nil } +func (store *LocalStore) GetRepository() string { + return store.repo +} + func (store *LocalStore) saveContentLayer(layer *artifact.ModelLayer) (*ocispec.Descriptor, error) { ctx := context.Background() diff --git a/pkg/lib/storage/store.go b/pkg/lib/storage/store.go index f33655f7..c70a28d5 100644 --- a/pkg/lib/storage/store.go +++ b/pkg/lib/storage/store.go @@ -13,6 +13,7 @@ import ( type Store interface { SaveModel(model *artifact.Model, tag string) (*ocispec.Descriptor, error) TagModel(manifestDesc ocispec.Descriptor, tag string) error + GetRepository() string ParseIndexJson() (*ocispec.Index, error) Fetch(context.Context, ocispec.Descriptor) ([]byte, error) } diff --git a/pkg/lib/testing/testing.go b/pkg/lib/testing/testing.go index 36a2c5e8..cff64ddb 100644 --- a/pkg/lib/testing/testing.go +++ b/pkg/lib/testing/testing.go @@ -19,6 +19,8 @@ var TestingNotFoundError = errors.New("artifact not found") var TestingInvalidSizeError = errors.New("invalid size") type TestStore struct { + // Repository for this store + Repo string // Map of digest to Manifest, to simulate retrieval from e.g. disk Manifests map[digest.Digest]ocispec.Manifest // Map of digest to Config, to simulate retrieval from e.g. disk @@ -77,3 +79,7 @@ func (*TestStore) TagModel(ocispec.Descriptor, string) error { func (*TestStore) SaveModel(*artifact.Model, string) (*ocispec.Descriptor, error) { return nil, fmt.Errorf("save model is not implemented for testing") } + +func (t *TestStore) GetRepository() string { + return t.Repo +}