support mkfs for userspace convertor

Signed-off-by: liulanzheng <[email protected]>

README.md

@@ -44,7 +44,7 @@ Accelerated Container Image is a __non-core__ sub-project of containerd.
 
     The convertor supports layer deduplication, which prevents duplication of layer conversion for every image conversion.
 
-* standalone userspace image-convertor (Experimental)
+* standalone userspace image-convertor
 
     Standalone userspace image-convertor has similar functionality to embedded image-convertor but runs in the userspace. It does not require root privilege and dependence on tcmu, configfs, snapshotter, or even on containerd. which makes it much more convenient to run in a container.
 

cmd/convertor/builder/builder.go

@@ -41,6 +41,7 @@ type BuilderOptions struct {
 	PlainHTTP bool
 	WorkDir   string
 	OCI       bool
+	Mkfs      bool
 	DB        database.ConversionDatabase
 	Engine    BuilderEngineType
 }
@@ -67,6 +68,7 @@ func NewOverlayBDBuilder(ctx context.Context, opt BuilderOptions) (Builder, erro
 	}
 	engineBase.workDir = opt.WorkDir
 	engineBase.oci = opt.OCI
+	engineBase.mkfs = opt.Mkfs
 	engineBase.db = opt.DB
 
 	refspec, err := reference.Parse(opt.Ref)

cmd/convertor/builder/builder_engine.go

@@ -70,6 +70,7 @@ type builderEngineBase struct {
 	config     specs.Image
 	workDir    string
 	oci        bool
+	mkfs       bool
 	db         database.ConversionDatabase
 	host       string
 	repository string

cmd/convertor/builder/overlaybd_builder.go

@@ -96,7 +96,7 @@ func (e *overlaybdBuilderEngine) BuildLayer(ctx context.Context, idx int) error
 		alreadyConverted = true
 	}
 	if !alreadyConverted {
-		if err := e.create(ctx, layerDir); err != nil {
+		if err := e.create(ctx, layerDir, e.mkfs && (idx == 0)); err != nil {
 			return err
 		}
 		e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{
@@ -146,12 +146,15 @@ func (e *overlaybdBuilderEngine) UploadImage(ctx context.Context) error {
 		e.manifest.Layers[idx] = e.overlaybdLayers[idx].desc
 		e.config.RootFS.DiffIDs[idx] = e.overlaybdLayers[idx].desc.Digest
 	}
-	baseDesc, err := e.uploadBaseLayer(ctx)
-	if err != nil {
-		return err
+	if !e.mkfs {
+		baseDesc, err := e.uploadBaseLayer(ctx)
+		if err != nil {
+			return err
+		}
+		e.manifest.Layers = append([]specs.Descriptor{baseDesc}, e.manifest.Layers...)
+		e.config.RootFS.DiffIDs = append([]digest.Digest{baseDesc.Digest}, e.config.RootFS.DiffIDs...)
 	}
-	e.manifest.Layers = append([]specs.Descriptor{baseDesc}, e.manifest.Layers...)
-	e.config.RootFS.DiffIDs = append([]digest.Digest{baseDesc.Digest}, e.config.RootFS.DiffIDs...)
+
 	return e.uploadManifestAndConfig(ctx)
 }
 
@@ -288,8 +291,13 @@ func (e *overlaybdBuilderEngine) getLayerDir(idx int) string {
 	return path.Join(e.workDir, fmt.Sprintf("%04d_", idx)+e.manifest.Layers[idx].Digest.String())
 }
 
-func (e *overlaybdBuilderEngine) create(ctx context.Context, dir string) error {
-	return utils.Create(ctx, dir, "-s", "64")
+func (e *overlaybdBuilderEngine) create(ctx context.Context, dir string, mkfs bool) error {
+	opts := []string{"-s", "64"}
+	if mkfs {
+		logrus.Info("do make ext4 fs")
+		opts = append(opts, "--mkfs")
+	}
+	return utils.Create(ctx, dir, opts...)
 }
 
 func (e *overlaybdBuilderEngine) apply(ctx context.Context, dir string) error {
@@ -304,7 +312,7 @@ func (e *overlaybdBuilderEngine) commit(ctx context.Context, dir string, idx int
 		parentUUID = ""
 	}
 	curUUID := chainIDtoUUID(e.overlaybdLayers[idx].chainID)
-	opts := []string{"-z", "-t", "--uuid", curUUID}
+	opts := []string{"-z", "--uuid", curUUID}
 	if parentUUID != "" {
 		opts = append(opts, "--parent-uuid", parentUUID)
 	}

cmd/convertor/builder/turboOCI_builder.go

@@ -84,7 +84,7 @@ func (e *turboOCIBuilderEngine) DownloadLayer(ctx context.Context, idx int) erro
 
 func (e *turboOCIBuilderEngine) BuildLayer(ctx context.Context, idx int) error {
 	layerDir := e.getLayerDir(idx)
-	if err := e.create(ctx, layerDir); err != nil {
+	if err := e.create(ctx, layerDir, e.mkfs && (idx == 0)); err != nil {
 		return err
 	}
 	e.overlaybdConfig.Upper = snapshot.OverlayBDBSConfigUpper{
@@ -226,8 +226,12 @@ func (e *turboOCIBuilderEngine) createIdentifier(idx int) error {
 	return nil
 }
 
-func (e *turboOCIBuilderEngine) create(ctx context.Context, dir string) error {
-	return utils.Create(ctx, dir, "-s", "64", "--turboOCI")
+func (e *turboOCIBuilderEngine) create(ctx context.Context, dir string, mkfs bool) error {
+	opts := []string{"-s", "64", "--turboOCI"}
+	if mkfs {
+		opts = append(opts, "--mkfs")
+	}
+	return utils.Create(ctx, dir, opts...)
 }
 
 func (e *turboOCIBuilderEngine) apply(ctx context.Context, dir string) error {

cmd/convertor/main.go

@@ -38,6 +38,7 @@ var (
 	tagOutput string
 	dir       string
 	oci       bool
+	mkfs      bool
 	verbose   bool
 	fastoci   string
 	turboOCI  string
@@ -75,6 +76,7 @@ var (
 				PlainHTTP: plain,
 				WorkDir:   dir,
 				OCI:       oci,
+				Mkfs:      mkfs,
 			}
 			if overlaybd != "" {
 				logrus.Info("building [Overlaybd - Native]  image...")
@@ -135,9 +137,10 @@ func init() {
 	rootCmd.Flags().BoolVarP(&plain, "plain", "", false, "connections using plain HTTP")
 	rootCmd.Flags().BoolVarP(&verbose, "verbose", "", false, "show debug log")
 	rootCmd.Flags().StringVarP(&tagInput, "input-tag", "i", "", "tag for image converting from (required)")
-	rootCmd.Flags().StringVarP(&tagOutput, "output-tag", "o", "", "tag for image converting to (required)")
+	rootCmd.Flags().StringVarP(&tagOutput, "output-tag", "o", "", "tag for image converting to")
 	rootCmd.Flags().StringVarP(&dir, "dir", "d", "tmp_conv", "directory used for temporary data")
 	rootCmd.Flags().BoolVarP(&oci, "oci", "", false, "export image with oci spec")
+	rootCmd.Flags().BoolVarP(&mkfs, "mkfs", "", false, "make ext4 fs in bottom layer")
 	rootCmd.Flags().StringVar(&fastoci, "fastoci", "", "build 'Overlaybd-Turbo OCIv1' format (old name of turboOCIv1. deprecated)")
 
 	rootCmd.Flags().StringVar(&turboOCI, "turboOCI", "", "build 'Overlaybd-Turbo OCIv1' format")

docs/USERSPACE_CONVERTOR.md

@@ -18,37 +18,41 @@ Only several tools are required:
 
 - baselayer
 
-  stored at `/opt/overlaybd/baselayers/ext4_64` after installing [overlaybd](https://github.com/containerd/overlaybd). This is only required just for now. Once the automatic mkfs is implemented, it's no longer needed.
+  stored at `/opt/overlaybd/baselayers/ext4_64` after installing [overlaybd](https://github.com/containerd/overlaybd). This is required if flag `--mkfs` is not provided.
 
-Overall, the requirements are `/opt/overlaybd/bin/{overlaybd-create,overlaybd-commit,overlaybd-apply}` and `/opt/overlaybd/baselayers/ext4_64`.
+Overall, the requirements are `/opt/overlaybd/bin/{overlaybd-create,overlaybd-commit,overlaybd-apply}` and `/opt/overlaybd/baselayers/ext4_64`(optional).
 
 ## Basic Usage
 
 ```bash
 # usage
 $ bin/convertor --help
 
-overlaybd-convertor is a standalone userspace image conversion tool that helps converting oci images to overlaybd images
+overlaybd convertor is a standalone userspace image conversion tool that helps converting oci images to overlaybd images
 
 Usage:
-  overlaybd-convertor [flags]
+  convertor [flags]
 
 Flags:
   -r, --repository string   repository for converting image (required)
   -u, --username string     user[:password] Registry user and password
       --plain               connections using plain HTTP
+      --verbose             show debug log
   -i, --input-tag string    tag for image converting from (required)
-  -o, --output-tag string   tag for image converting to (required)
+  -o, --output-tag string   tag for image converting to
   -d, --dir string          directory used for temporary data (default "tmp_conv")
-  -h, --help                help for overlaybd-convertor
       --oci                 export image with oci spec
-      --fastoci             build fastoci format
-      --overlaybd           build overlaybd format
-      --db-str              db str for overlaybd conversion
-      --db-type             type of db to use for conversion deduplication. Available: mysql. Default none
+      --mkfs                make ext4 fs in bottom layer
+      --fastoci string      build 'Overlaybd-Turbo OCIv1' format (old name of turboOCIv1. deprecated)
+      --turboOCI string     build 'Overlaybd-Turbo OCIv1' format
+      --overlaybd string    build overlaybd format
+      --db-str string       db str for overlaybd conversion
+      --db-type string      type of db to use for conversion deduplication. Available: mysql. Default none
+  -h, --help                help for convertor
 
 # examples
 $ bin/convertor -r docker.io/overlaybd/redis -u user:pass -i 6.2.6 -o 6.2.6_obd
+$ bin/convertor --mkfs -r docker.io/overlaybd/redis -u user:pass -i 6.2.6 --overlaybd 6.2.6_obd
 $ bin/convertor -r docker.io/overlaybd/redis -u user:pass -i 6.2.6 --overlaybd 6.2.6_obd --fastoci 6.2.6_foci
 
 ```