Merge pull request #495 from zong-zhe/refactor-vendor

feat: make vendor supports mvs

pkg/client/client.go

@@ -13,7 +13,6 @@ import (
 	"github.com/dominikbraun/graph"
 	"github.com/elliotchance/orderedmap/v2"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/otiai10/copy"
 	"golang.org/x/mod/module"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"oras.land/oras-go/pkg/auth"
@@ -668,105 +667,6 @@ func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool)
 	return nil
 }
 
-func (c *KpmClient) vendorDeps(kclPkg *pkg.KclPkg, vendorPath string) error {
-	lockDeps := make([]pkg.Dependency, 0, kclPkg.Dependencies.Deps.Len())
-	for _, k := range kclPkg.Dependencies.Deps.Keys() {
-		d, _ := kclPkg.Dependencies.Deps.Get(k)
-		lockDeps = append(lockDeps, d)
-	}
-
-	// Traverse all dependencies in kcl.mod.lock.
-	for i := 0; i < len(lockDeps); i++ {
-		d := lockDeps[i]
-		if len(d.Name) == 0 {
-			return errors.InvalidDependency
-		}
-		// If the dependency is from the local path, do not vendor it, vendor its dependencies.
-		if d.IsFromLocal() {
-			dpkg, err := c.LoadPkgFromPath(d.GetLocalFullPath(kclPkg.HomePath))
-			if err != nil {
-				return err
-			}
-			err = c.vendorDeps(dpkg, vendorPath)
-			if err != nil {
-				return err
-			}
-			continue
-		} else {
-			vendorFullPath := filepath.Join(vendorPath, d.GenPathSuffix())
-
-			// If the package already exists in the 'vendor', do nothing.
-			if utils.DirExists(vendorFullPath) {
-				d.LocalFullPath = vendorFullPath
-				lockDeps[i] = d
-				continue
-			} else {
-				// If not in the 'vendor', check the global cache.
-				cacheFullPath := c.getDepStorePath(c.homePath, &d, false)
-				if utils.DirExists(cacheFullPath) {
-					// If there is, copy it into the 'vendor' directory.
-					err := copy.Copy(cacheFullPath, vendorFullPath)
-					if err != nil {
-						return err
-					}
-				} else {
-					// re-download if not.
-					err := c.AddDepToPkg(kclPkg, &d)
-					if err != nil {
-						return err
-					}
-					// re-vendor again with new kcl.mod and kcl.mod.lock
-					err = c.vendorDeps(kclPkg, vendorPath)
-					if err != nil {
-						return err
-					}
-					return nil
-				}
-			}
-
-			if d.GetPackage() != "" {
-				tempVendorFullPath, err := utils.FindPackage(vendorFullPath, d.GetPackage())
-				if err != nil {
-					return err
-				}
-				vendorFullPath = tempVendorFullPath
-			}
-
-			dpkg, err := c.LoadPkgFromPath(vendorFullPath)
-			if err != nil {
-				return err
-			}
-
-			// Vendor the dependencies of the current dependency.
-			err = c.vendorDeps(dpkg, vendorPath)
-			if err != nil {
-				return err
-			}
-			d.LocalFullPath = vendorFullPath
-			lockDeps[i] = d
-		}
-	}
-
-	// Update the dependencies in kcl.mod.lock.
-	for _, d := range lockDeps {
-		kclPkg.Dependencies.Deps.Set(d.Name, d)
-	}
-
-	return nil
-}
-
-// VendorDeps will vendor all the dependencies of the current kcl package.
-func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error {
-	// Mkdir the dir "vendor".
-	vendorPath := kclPkg.LocalVendorPath()
-	err := os.MkdirAll(vendorPath, 0755)
-	if err != nil {
-		return err
-	}
-
-	return c.vendorDeps(kclPkg, vendorPath)
-}
-
 // FillDepInfo will fill registry information for a dependency.
 func (c *KpmClient) FillDepInfo(dep *pkg.Dependency, homepath string) error {
 	// Homepath for a dependency is the homepath of the kcl package.
@@ -1428,9 +1328,9 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc
 }
 
 // dependencyExists will check whether the dependency exists in the local filesystem.
-func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency) (*pkg.Dependency, error) {
+func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency, isVendor bool) (*pkg.Dependency, error) {
 	// If the flag '--no_sum_check' is set, skip the checksum check.
-	deppath := c.getDepStorePath(searchPath, dep, false)
+	deppath := c.getDepStorePath(searchPath, dep, isVendor)
 	if utils.DirExists(deppath) {
 		depPkg, err := c.LoadPkgFromPath(deppath)
 		if err != nil {
@@ -1440,6 +1340,13 @@ func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency
 		// TODO: new local dependency structure will replace this
 		// issue: https://github.com/kcl-lang/kpm/issues/384
 		dep.FullName = dep.GenDepFullName()
+
+		if dep.GetPackage() != "" {
+			dep.LocalFullPath, err = utils.FindPackage(dep.LocalFullPath, dep.GetPackage())
+			if err != nil {
+				return nil, err
+			}
+		}
 		return dep, nil
 	}
 	return nil, nil
@@ -1459,7 +1366,7 @@ func (c *KpmClient) DownloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependenc
 			return nil, errors.InvalidDependency
 		}
 
-		existDep, err := c.dependencyExistsLocal(pkghome, &d)
+		existDep, err := c.dependencyExistsLocal(pkghome, &d, false)
 		if existDep != nil && err == nil {
 			newDeps.Deps.Set(d.Name, *existDep)
 			continue

pkg/client/client_test.go

@@ -75,7 +75,6 @@ func TestWithGlobalLock(t *testing.T) {
 	test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStr", testResolveMetadataInJsonStr)
 	test.RunTestWithGlobalLock(t, "testPackageCurrentPkgPath", testPackageCurrentPkgPath)
 	test.RunTestWithGlobalLock(t, "TestUpdateKclModAndLock", testUpdateKclModAndLock)
-	test.RunTestWithGlobalLock(t, "TestVendorDeps", testVendorDeps)
 	test.RunTestWithGlobalLock(t, "TestResolveDepsWithOnlyKclMod", testResolveDepsWithOnlyKclMod)
 	test.RunTestWithGlobalLock(t, "TestResolveDepsVendorMode", testResolveDepsVendorMode)
 	test.RunTestWithGlobalLock(t, "TestCompileWithEntryFile", testCompileWithEntryFile)
@@ -528,68 +527,6 @@ func testUpdateKclModAndLock(t *testing.T) {
 	}
 }
 
-func testVendorDeps(t *testing.T) {
-	testDir := getTestDir("resolve_deps")
-	kpm_home := filepath.Join(testDir, "kpm_home")
-	os.RemoveAll(filepath.Join(testDir, "my_kcl"))
-	kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1"))
-	kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2"))
-
-	depKcl1 := pkg.Dependency{
-		Name:     "kcl1",
-		FullName: "kcl1",
-		Version:  "0.0.1",
-		Sum:      kcl1Sum,
-	}
-
-	depKcl2 := pkg.Dependency{
-		Name:     "kcl2",
-		FullName: "kcl2",
-		Version:  "0.0.1",
-		Sum:      kcl2Sum,
-	}
-
-	mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]()
-	mppTest.Set("kcl1", depKcl1)
-	mppTest.Set("kcl2", depKcl2)
-
-	kclPkg := pkg.KclPkg{
-		ModFile: pkg.ModFile{
-			HomePath: filepath.Join(testDir, "my_kcl"),
-			// Whether the current package uses the vendor mode
-			// In the vendor mode, kpm will look for the package in the vendor subdirectory
-			// in the current package directory.
-			VendorMode: false,
-			Dependencies: pkg.Dependencies{
-				Deps: mppTest,
-			},
-		},
-		HomePath: filepath.Join(testDir, "my_kcl"),
-		// The dependencies in the current kcl package are the dependencies of kcl.mod.lock,
-		// not the dependencies in kcl.mod.
-		Dependencies: pkg.Dependencies{
-			Deps: mppTest,
-		},
-	}
-
-	mykclVendorPath := filepath.Join(filepath.Join(testDir, "my_kcl"), "vendor")
-	assert.Equal(t, utils.DirExists(mykclVendorPath), false)
-	kpmcli, err := NewKpmClient()
-	kpmcli.homePath = kpm_home
-	assert.Equal(t, err, nil)
-	err = kpmcli.VendorDeps(&kclPkg)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, utils.DirExists(mykclVendorPath), true)
-	assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1_0.0.1")), true)
-	assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2_0.0.1")), true)
-
-	maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg)
-	assert.Equal(t, err, nil)
-	assert.Equal(t, len(maps), 2)
-
-	os.RemoveAll(filepath.Join(testDir, "my_kcl"))
-}
-
 func testResolveDepsWithOnlyKclMod(t *testing.T) {
 	testDir := getTestDir("resolve_dep_with_kclmod")
 	assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), false)
@@ -622,13 +559,27 @@ func testResolveDepsVendorMode(t *testing.T) {
 		FullName: "kcl1_0.0.1",
 		Version:  "0.0.1",
 		Sum:      kcl1Sum,
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/kcl1",
+				Tag:  "0.0.1",
+			},
+		},
 	}
 
 	depKcl2 := pkg.Dependency{
 		Name:     "kcl2",
 		FullName: "kcl2_0.0.1",
 		Version:  "0.0.1",
 		Sum:      kcl2Sum,
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/kcl2",
+				Tag:  "0.0.1",
+			},
+		},
 	}
 
 	mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]()
@@ -687,13 +638,27 @@ func testCompileWithEntryFile(t *testing.T) {
 		FullName: "kcl1_0.0.1",
 		Version:  "0.0.1",
 		Sum:      kcl1Sum,
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/kcl2",
+				Tag:  "0.0.1",
+			},
+		},
 	}
 	kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2"))
 	depKcl2 := pkg.Dependency{
 		Name:     "kcl2",
 		FullName: "kcl2_0.0.1",
 		Version:  "0.0.1",
 		Sum:      kcl2Sum,
+		Source: downloader.Source{
+			Oci: &downloader.Oci{
+				Reg:  "ghcr.io",
+				Repo: "kcl-lang/kcl2",
+				Tag:  "0.0.1",
+			},
+		},
 	}
 
 	mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]()
@@ -796,7 +761,7 @@ func testResolveMetadataInJsonStr(t *testing.T) {
 	expectedDep.Deps["flask_demo_kcl_manifests"] = pkg.Dependency{
 		Name:          "flask_demo_kcl_manifests",
 		FullName:      "flask-demo-kcl-manifests_ade147b",
-		Version:       "ade147b",
+		Version:       "0.1.0",
 		LocalFullPath: filepath.Join(globalPkgPath, "flask-demo-kcl-manifests_ade147b"),
 	}
 
@@ -819,7 +784,7 @@ func testResolveMetadataInJsonStr(t *testing.T) {
 	expectedDep.Deps["flask_demo_kcl_manifests"] = pkg.Dependency{
 		Name:          "flask_demo_kcl_manifests",
 		FullName:      "flask-demo-kcl-manifests_ade147b",
-		Version:       "ade147b",
+		Version:       "0.1.0",
 		LocalFullPath: filepath.Join(vendorDir, "flask-demo-kcl-manifests_ade147b"),
 	}
 
@@ -869,13 +834,13 @@ func testResolveMetadataInJsonStrWithPackage(t *testing.T) {
 		Deps: make(map[string]pkg.Dependency),
 	}
 
-	localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld")
+	localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "flask-demo-kcl-manifests_8308200"), "cc")
 	assert.Equal(t, err, nil)
 
-	expectedDep.Deps["helloworld"] = pkg.Dependency{
-		Name:          "helloworld",
-		FullName:      "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8",
-		Version:       "ee03122b5f45b09eb48694422fc99a0772f6bba8",
+	expectedDep.Deps["cc"] = pkg.Dependency{
+		Name:          "cc",
+		FullName:      "flask-demo-kcl-manifests_8308200",
+		Version:       "8308200",
 		LocalFullPath: localFullPath,
 	}
 
@@ -903,19 +868,19 @@ func testResolveMetadataInJsonStrWithPackage(t *testing.T) {
 	assert.Equal(t, err, nil)
 
 	assert.Equal(t, utils.DirExists(vendorDir), true)
-	assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8")), true)
+	assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200")), true)
 
-	localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld")
+	localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200"), "cc")
 	assert.Equal(t, err, nil)
 
 	expectedDep = pkg.DependenciesUI{
 		Deps: make(map[string]pkg.Dependency),
 	}
 
-	expectedDep.Deps["helloworld"] = pkg.Dependency{
-		Name:          "helloworld",
-		FullName:      "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8",
-		Version:       "ee03122b5f45b09eb48694422fc99a0772f6bba8",
+	expectedDep.Deps["cc"] = pkg.Dependency{
+		Name:          "cc",
+		FullName:      "flask-demo-kcl-manifests_8308200",
+		Version:       "8308200",
 		LocalFullPath: localFullPath,
 	}
 

pkg/client/test_data/resolve_metadata/with_package/kcl.mod

@@ -4,4 +4,4 @@ edition = "v0.8.0"
 version = "0.0.1"
 
 [dependencies]
-helloworld = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "helloworld" }
+cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" }

pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock

@@ -1,8 +1,8 @@
 [dependencies]
-  [dependencies.helloworld]
-    name = "helloworld"
-    full_name = "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"
-    version = "0.1.2"
-    url = "https://github.com/kcl-lang/modules.git"
-    commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8"
-    package = "helloworld"
+  [dependencies.cc]
+    name = "cc"
+    full_name = "flask-demo-kcl-manifests_8308200"
+    version = "0.0.1"
+    url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git"
+    commit = "8308200"
+    package = "cc"

pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod

@@ -0,0 +1,7 @@
+[package]
+name = "dep1"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]
+helloworld = "0.1.1"

pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod.lock

@@ -0,0 +1,5 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.1"
+    version = "0.1.1"

pkg/client/test_data/test_vendor_mvs/dep1/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod

@@ -0,0 +1,7 @@
+[package]
+name = "dep2"
+edition = "v0.10.0"
+version = "0.0.1"
+
+[dependencies]
+helloworld = "0.1.2"

pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod.lock

@@ -0,0 +1,5 @@
+[dependencies]
+  [dependencies.helloworld]
+    name = "helloworld"
+    full_name = "helloworld_0.1.2"
+    version = "0.1.2"