From 457ab34e32f439ff89debfc1b4424919b6029b26 Mon Sep 17 00:00:00 2001 From: Angel Misevski Date: Fri, 9 Feb 2024 16:41:46 -0500 Subject: [PATCH 1/3] Add support for tagging builds Add support for applying a tag when building via the --tag|-t flag. If supplied, the build will be labelled with the tag. As an early implementation, this will store the model's blobs _in a different path_ than the default, matching the tag. For a tag like myregistry.com/my-organization/my-repo:my-tag and a root config path of $HOME/.jozu/, blobs will be stored in $HOME/.jozu/storage/myregistry.com/my-organization/my-repo/ This means that building the same model with different tags will duplicate storage for now; this should be fixed in the future. Summary of changes in this commit: * Storage path is no longer .jozuStore appended to whatever config path is; instead Storage uses whatever path is provided and build command chooses 'storage' subdirecto of the config path * This is required to allow all tagged repos to be subdirectories of one base 'storage' directory * If no tag is provided, blobs are stored in $JOZU_HOME/storage/ as before (change: storage instead of .jozuStore) * If a tag is provided, it is applied as an annotation according to the OCI spec, using annotation 'org.opencontainers.image.ref.name' * If a tag is provided but does not have a hostname, the default value of localhost is used. --- pkg/cmd/build/build.go | 43 +++++++++++++++++++++++++++++++--------- pkg/lib/storage/local.go | 27 +++++++++++++++---------- 2 files changed, 50 insertions(+), 20 deletions(-) diff --git a/pkg/cmd/build/build.go b/pkg/cmd/build/build.go index 00534010..78c2893b 100644 --- a/pkg/cmd/build/build.go +++ b/pkg/cmd/build/build.go @@ -6,6 +6,8 @@ package build import ( "fmt" "os" + "path" + "strings" "jmm/pkg/artifact" "jmm/pkg/lib/storage" @@ -13,9 +15,12 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/spf13/viper" + "oras.land/oras-go/v2/registry" ) -const DEFAULT_MODEL_FILE = "Jozufile" +const ( + DEFAULT_MODEL_FILE = "Jozufile" +) var ( shortDesc = `Build a model` @@ -28,13 +33,16 @@ var ( ) type BuildFlags struct { - ModelFile string + ModelFile string + FullTagRef string } type BuildOptions struct { - ModelFile string - ContextDir string - JozuHome string + ModelFile string + ContextDir string + configHome string + storageHome string + modelRef *registry.Reference } func NewCmdBuild() *cobra.Command { @@ -76,8 +84,9 @@ func (options *BuildOptions) Complete(cmd *cobra.Command, argsIn []string) error if options.ModelFile == "" { options.ModelFile = options.ContextDir + "/" + DEFAULT_MODEL_FILE } - fmt.Println("config: ", viper.GetString("config")) - options.JozuHome = viper.GetString("config") + options.configHome = viper.GetString("config") + fmt.Println("config: ", options.configHome) + options.storageHome = path.Join(options.configHome, "storage") return nil } @@ -107,7 +116,12 @@ func (options *BuildOptions) RunBuild() error { model.Layers = append(model.Layers, layer) model.Config = jozufile - store := storage.NewLocalStore(options.JozuHome) + modelStorePath := options.storageHome + if options.modelRef != nil { + modelStorePath = path.Join(options.storageHome, options.modelRef.Registry, options.modelRef.Repository) + model.Tag = options.modelRef.Reference + } + store := storage.NewLocalStore(modelStorePath) var manifest *v1.Manifest manifest, err = store.SaveModel(model) if err != nil { @@ -123,13 +137,24 @@ func (o *BuildFlags) ToOptions() (*BuildOptions, error) { if o.ModelFile != "" { options.ModelFile = o.ModelFile } + if o.FullTagRef != "" { + // References _must_ contain host; use localhost to mark local-only + if !strings.Contains(o.FullTagRef, "/") { + o.FullTagRef = fmt.Sprintf("localhost/%s", o.FullTagRef) + } + modelRef, err := registry.ParseReference(o.FullTagRef) + if err != nil { + return nil, err + } + options.modelRef = &modelRef + } return options, nil } func (flags *BuildFlags) AddFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&flags.ModelFile, "file", "f", "", "Path to the model file") + cmd.Flags().StringVarP(&flags.FullTagRef, "tag", "t", "", "Tag for the model") cmd.Args = cobra.ExactArgs(1) - } func NewBuildFlags() *BuildFlags { diff --git a/pkg/lib/storage/local.go b/pkg/lib/storage/local.go index 65b644c1..16f06951 100644 --- a/pkg/lib/storage/local.go +++ b/pkg/lib/storage/local.go @@ -18,15 +18,14 @@ import ( ) type LocalStore struct { - Storage *oci.Store + storage *oci.Store indexPath string } // Assert LocalStore implements the Store interface. var _ Store = (*LocalStore)(nil) -func NewLocalStore(jozuhome string) *LocalStore { - storeHome := filepath.Join(jozuhome, ".jozuStore") +func NewLocalStore(storeHome string) Store { indexPath := filepath.Join(storeHome, "index.json") store, err := oci.New(storeHome) @@ -35,7 +34,7 @@ func NewLocalStore(jozuhome string) *LocalStore { } return &LocalStore{ - Storage: store, + storage: store, indexPath: indexPath, } } @@ -60,7 +59,7 @@ func (store *LocalStore) SaveModel(model *artifact.Model) (*ocispec.Manifest, er } func (store *LocalStore) Fetch(ctx context.Context, desc ocispec.Descriptor) ([]byte, error) { - bytes, err := content.FetchAll(ctx, store.Storage, desc) + bytes, err := content.FetchAll(ctx, store.storage, desc) return bytes, err } @@ -93,7 +92,7 @@ func (store *LocalStore) saveContentLayer(layer *artifact.ModelLayer) (*ocispec. Digest: digest.FromBytes(buf.Bytes()), Size: int64(buf.Len()), } - err = store.Storage.Push(ctx, desc, buf) + err = store.storage.Push(ctx, desc, buf) layer.Descriptor = desc if err != nil { return nil, err @@ -113,7 +112,7 @@ func (store *LocalStore) saveConfigFile(model *artifact.JozuFile) (*ocispec.Desc Digest: digest.FromBytes(buf), Size: int64(len(buf)), } - err = store.Storage.Push(ctx, desc, bytes.NewReader(buf)) + err = store.storage.Push(ctx, desc, bytes.NewReader(buf)) if err != nil { return nil, err } @@ -123,15 +122,21 @@ func (store *LocalStore) saveConfigFile(model *artifact.JozuFile) (*ocispec.Desc func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispec.Descriptor) (*ocispec.Manifest, error) { ctx := context.Background() manifest := ocispec.Manifest{ - Versioned: specs.Versioned{SchemaVersion: 2}, - Config: *config, - Layers: []ocispec.Descriptor{}, + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: *config, + Layers: []ocispec.Descriptor{}, + Annotations: map[string]string{}, } // Add the layers to the manifest for _, layer := range model.Layers { manifest.Layers = append(manifest.Layers, layer.Descriptor) } + // Add tags, if any + if model.Tag != "" { + manifest.Annotations[ocispec.AnnotationRefName] = model.Tag + } + manifestBytes, err := json.Marshal(manifest) if err != nil { return nil, err @@ -142,7 +147,7 @@ func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispe Digest: digest.FromBytes(manifestBytes), Size: int64(len(manifestBytes)), } - err = store.Storage.Push(ctx, desc, bytes.NewReader(manifestBytes)) + err = store.storage.Push(ctx, desc, bytes.NewReader(manifestBytes)) if err != nil { return nil, err } From a2d42ec38ca793d39bdbeab94bea89ca744bd8ff Mon Sep 17 00:00:00 2001 From: Angel Misevski Date: Fri, 9 Feb 2024 17:03:57 -0500 Subject: [PATCH 2/3] Fixup build to check if content already exists in store before push Avoid error when pushing e.g. an already-existing blob into the local storage. --- pkg/lib/storage/local.go | 44 ++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/lib/storage/local.go b/pkg/lib/storage/local.go index 16f06951..3ab52195 100644 --- a/pkg/lib/storage/local.go +++ b/pkg/lib/storage/local.go @@ -92,30 +92,49 @@ func (store *LocalStore) saveContentLayer(layer *artifact.ModelLayer) (*ocispec. Digest: digest.FromBytes(buf.Bytes()), Size: int64(buf.Len()), } - err = store.storage.Push(ctx, desc, buf) - layer.Descriptor = desc + + exists, err := store.storage.Exists(ctx, desc) if err != nil { return nil, err } - fmt.Println("Saved model layer: ", desc.Digest) + if exists { + fmt.Println("Model layer already saved: ", desc.Digest) + } else { + // Does not exist in storage, need to push + err = store.storage.Push(ctx, desc, buf) + if err != nil { + return nil, err + } + fmt.Println("Saved model layer: ", desc.Digest) + } + return &desc, nil } func (store *LocalStore) saveConfigFile(model *artifact.JozuFile) (*ocispec.Descriptor, error) { ctx := context.Background() - buf, err := model.MarshalToJSON() + modelBytes, err := model.MarshalToJSON() if err != nil { return nil, err } desc := ocispec.Descriptor{ MediaType: constants.ModelConfigMediaType, - Digest: digest.FromBytes(buf), - Size: int64(len(buf)), + Digest: digest.FromBytes(modelBytes), + Size: int64(len(modelBytes)), } - err = store.storage.Push(ctx, desc, bytes.NewReader(buf)) + + exists, err := store.storage.Exists(ctx, desc) if err != nil { return nil, err } + if !exists { + // Does not exist in storage, need to push + err = store.storage.Push(ctx, desc, bytes.NewReader(modelBytes)) + if err != nil { + return nil, err + } + } + return &desc, nil } @@ -147,9 +166,16 @@ func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispe Digest: digest.FromBytes(manifestBytes), Size: int64(len(manifestBytes)), } - err = store.storage.Push(ctx, desc, bytes.NewReader(manifestBytes)) - if err != nil { + + if exists, err := store.storage.Exists(ctx, desc); err != nil { return nil, err + } else if !exists { + // Does not exist in storage, need to push + err = store.storage.Push(ctx, desc, bytes.NewReader(manifestBytes)) + if err != nil { + return nil, err + } } + return &manifest, nil } From fb913a4261762a7ae14529c8b6b3943f15d2ae8d Mon Sep 17 00:00:00 2001 From: Angel Misevski Date: Mon, 12 Feb 2024 10:43:53 -0500 Subject: [PATCH 3/3] Fixup how tagging is done; add support for comma-separated tags Fix how tagging is done; instead of tagging manifests themselves, we need to tag their entries in the index.json. Additionally, add support for multiple tags specified at once, e.g. jmm build -t myreg/myrepo:tag1,tag2,tag3 --- pkg/artifact/model.go | 3 +-- pkg/cmd/build/build.go | 28 ++++++++++++++++++++-------- pkg/lib/storage/common.go | 17 +++++++++++++++++ pkg/lib/storage/local.go | 38 +++++++++++++++++++++++++++----------- pkg/lib/storage/store.go | 3 ++- pkg/lib/testing/testing.go | 6 +++++- 6 files changed, 72 insertions(+), 23 deletions(-) create mode 100644 pkg/lib/storage/common.go diff --git a/pkg/artifact/model.go b/pkg/artifact/model.go index cb33e47a..c2faaff6 100644 --- a/pkg/artifact/model.go +++ b/pkg/artifact/model.go @@ -2,7 +2,6 @@ package artifact type Model struct { Repository string - Tag string - Layers []*ModelLayer + Layers []ModelLayer Config *JozuFile } diff --git a/pkg/cmd/build/build.go b/pkg/cmd/build/build.go index 78c2893b..4c5529d0 100644 --- a/pkg/cmd/build/build.go +++ b/pkg/cmd/build/build.go @@ -12,7 +12,6 @@ import ( "jmm/pkg/artifact" "jmm/pkg/lib/storage" - v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/spf13/cobra" "github.com/spf13/viper" "oras.land/oras-go/v2/registry" @@ -43,6 +42,7 @@ type BuildOptions struct { configHome string storageHome string modelRef *registry.Reference + extraRefs []string } func NewCmdBuild() *cobra.Command { @@ -113,22 +113,30 @@ func (options *BuildOptions) RunBuild() error { // 3. Tar the build context and push to local registry layer := artifact.NewLayer(options.ContextDir) model := &artifact.Model{} - model.Layers = append(model.Layers, layer) + model.Layers = append(model.Layers, *layer) model.Config = jozufile modelStorePath := options.storageHome + tag := "" if options.modelRef != nil { modelStorePath = path.Join(options.storageHome, options.modelRef.Registry, options.modelRef.Repository) - model.Tag = options.modelRef.Reference + tag = options.modelRef.Reference } store := storage.NewLocalStore(modelStorePath) - var manifest *v1.Manifest - manifest, err = store.SaveModel(model) + manifestDesc, err := store.SaveModel(model, tag) if err != nil { fmt.Println(err) return err } - fmt.Println("Model saved: ", manifest.Config.Digest) + + for _, tag := range options.extraRefs { + if err := store.TagModel(*manifestDesc, tag); err != nil { + return err + } + } + + fmt.Println("Model saved: ", manifestDesc.Digest) + return nil } @@ -142,18 +150,22 @@ func (o *BuildFlags) ToOptions() (*BuildOptions, error) { if !strings.Contains(o.FullTagRef, "/") { o.FullTagRef = fmt.Sprintf("localhost/%s", o.FullTagRef) } - modelRef, err := registry.ParseReference(o.FullTagRef) + + // Handle multiple tags specified with commas, e.g. /:tag1,tag2 + refs := strings.Split(o.FullTagRef, ",") + modelRef, err := registry.ParseReference(refs[0]) if err != nil { return nil, err } options.modelRef = &modelRef + options.extraRefs = refs[1:] } return options, nil } func (flags *BuildFlags) AddFlags(cmd *cobra.Command) { cmd.Flags().StringVarP(&flags.ModelFile, "file", "f", "", "Path to the model file") - cmd.Flags().StringVarP(&flags.FullTagRef, "tag", "t", "", "Tag for the model") + cmd.Flags().StringVarP(&flags.FullTagRef, "tag", "t", "", "Tag for the model. Example: -t registry/repository:tag1,tag2") cmd.Args = cobra.ExactArgs(1) } diff --git a/pkg/lib/storage/common.go b/pkg/lib/storage/common.go new file mode 100644 index 00000000..814161ea --- /dev/null +++ b/pkg/lib/storage/common.go @@ -0,0 +1,17 @@ +package storage + +import ( + "fmt" + "regexp" +) + +var ( + validTagRegex = regexp.MustCompile(`^[a-zA-Z0-9_][a-zA-Z0-9._-]{0,127}$`) +) + +func validateTag(tag string) error { + if !validTagRegex.MatchString(tag) { + return fmt.Errorf("invalid tag") + } + return nil +} diff --git a/pkg/lib/storage/local.go b/pkg/lib/storage/local.go index 3ab52195..7e11e115 100644 --- a/pkg/lib/storage/local.go +++ b/pkg/lib/storage/local.go @@ -39,23 +39,35 @@ func NewLocalStore(storeHome string) Store { } } -func (store *LocalStore) SaveModel(model *artifact.Model) (*ocispec.Manifest, error) { +func (store *LocalStore) SaveModel(model *artifact.Model, tag string) (*ocispec.Descriptor, error) { config, err := store.saveConfigFile(model.Config) if err != nil { return nil, err } for _, layer := range model.Layers { - _, err = store.saveContentLayer(layer) + _, err = store.saveContentLayer(&layer) if err != nil { return nil, err } } - manifest, err := store.saveModelManifest(model, config) + manifestDesc, err := store.saveModelManifest(model, config, tag) if err != nil { return nil, err } - return manifest, nil + return manifestDesc, nil +} + +func (store *LocalStore) TagModel(manifestDesc ocispec.Descriptor, tag string) error { + if err := validateTag(tag); err != nil { + return err + } + + if err := store.storage.Tag(context.Background(), manifestDesc, tag); err != nil { + return fmt.Errorf("failed to tag manifest: %w", err) + } + + return nil } func (store *LocalStore) Fetch(ctx context.Context, desc ocispec.Descriptor) ([]byte, error) { @@ -138,7 +150,7 @@ func (store *LocalStore) saveConfigFile(model *artifact.JozuFile) (*ocispec.Desc return &desc, nil } -func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispec.Descriptor) (*ocispec.Manifest, error) { +func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispec.Descriptor, tag string) (*ocispec.Descriptor, error) { ctx := context.Background() manifest := ocispec.Manifest{ Versioned: specs.Versioned{SchemaVersion: 2}, @@ -151,11 +163,6 @@ func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispe manifest.Layers = append(manifest.Layers, layer.Descriptor) } - // Add tags, if any - if model.Tag != "" { - manifest.Annotations[ocispec.AnnotationRefName] = model.Tag - } - manifestBytes, err := json.Marshal(manifest) if err != nil { return nil, err @@ -177,5 +184,14 @@ func (store *LocalStore) saveModelManifest(model *artifact.Model, config *ocispe } } - return &manifest, nil + if tag != "" { + if err := validateTag(tag); err != nil { + return nil, err + } + if err := store.storage.Tag(context.Background(), desc, tag); err != nil { + return nil, fmt.Errorf("failed to tag manifest: %w", err) + } + } + + return &desc, nil } diff --git a/pkg/lib/storage/store.go b/pkg/lib/storage/store.go index 5d68ff8d..f33655f7 100644 --- a/pkg/lib/storage/store.go +++ b/pkg/lib/storage/store.go @@ -11,7 +11,8 @@ import ( ) type Store interface { - SaveModel(*artifact.Model) (*ocispec.Manifest, error) + SaveModel(model *artifact.Model, tag string) (*ocispec.Descriptor, error) + TagModel(manifestDesc ocispec.Descriptor, tag string) error ParseIndexJson() (*ocispec.Index, error) Fetch(context.Context, ocispec.Descriptor) ([]byte, error) } diff --git a/pkg/lib/testing/testing.go b/pkg/lib/testing/testing.go index 84e3618d..36a2c5e8 100644 --- a/pkg/lib/testing/testing.go +++ b/pkg/lib/testing/testing.go @@ -69,7 +69,11 @@ func (s *TestStore) ParseIndexJson() (*ocispec.Index, error) { return nil, TestingNotFoundError } +func (*TestStore) TagModel(ocispec.Descriptor, string) error { + return fmt.Errorf("tag model is not implemented for testing") +} + // SaveModel is not yet implemented! -func (*TestStore) SaveModel(*artifact.Model) (*ocispec.Manifest, error) { +func (*TestStore) SaveModel(*artifact.Model, string) (*ocispec.Descriptor, error) { return nil, fmt.Errorf("save model is not implemented for testing") }