diff --git a/pkg/client/client.go b/pkg/client/client.go index 514eee97..c002e071 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -13,7 +13,6 @@ import ( "github.com/dominikbraun/graph" "github.com/elliotchance/orderedmap/v2" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/otiai10/copy" "golang.org/x/mod/module" "kcl-lang.io/kcl-go/pkg/kcl" "oras.land/oras-go/pkg/auth" @@ -668,105 +667,6 @@ func (c *KpmClient) Package(kclPkg *pkg.KclPkg, tarPath string, vendorMode bool) return nil } -func (c *KpmClient) vendorDeps(kclPkg *pkg.KclPkg, vendorPath string) error { - lockDeps := make([]pkg.Dependency, 0, kclPkg.Dependencies.Deps.Len()) - for _, k := range kclPkg.Dependencies.Deps.Keys() { - d, _ := kclPkg.Dependencies.Deps.Get(k) - lockDeps = append(lockDeps, d) - } - - // Traverse all dependencies in kcl.mod.lock. - for i := 0; i < len(lockDeps); i++ { - d := lockDeps[i] - if len(d.Name) == 0 { - return errors.InvalidDependency - } - // If the dependency is from the local path, do not vendor it, vendor its dependencies. - if d.IsFromLocal() { - dpkg, err := c.LoadPkgFromPath(d.GetLocalFullPath(kclPkg.HomePath)) - if err != nil { - return err - } - err = c.vendorDeps(dpkg, vendorPath) - if err != nil { - return err - } - continue - } else { - vendorFullPath := filepath.Join(vendorPath, d.GenPathSuffix()) - - // If the package already exists in the 'vendor', do nothing. - if utils.DirExists(vendorFullPath) { - d.LocalFullPath = vendorFullPath - lockDeps[i] = d - continue - } else { - // If not in the 'vendor', check the global cache. - cacheFullPath := c.getDepStorePath(c.homePath, &d, false) - if utils.DirExists(cacheFullPath) { - // If there is, copy it into the 'vendor' directory. - err := copy.Copy(cacheFullPath, vendorFullPath) - if err != nil { - return err - } - } else { - // re-download if not. - err := c.AddDepToPkg(kclPkg, &d) - if err != nil { - return err - } - // re-vendor again with new kcl.mod and kcl.mod.lock - err = c.vendorDeps(kclPkg, vendorPath) - if err != nil { - return err - } - return nil - } - } - - if d.GetPackage() != "" { - tempVendorFullPath, err := utils.FindPackage(vendorFullPath, d.GetPackage()) - if err != nil { - return err - } - vendorFullPath = tempVendorFullPath - } - - dpkg, err := c.LoadPkgFromPath(vendorFullPath) - if err != nil { - return err - } - - // Vendor the dependencies of the current dependency. - err = c.vendorDeps(dpkg, vendorPath) - if err != nil { - return err - } - d.LocalFullPath = vendorFullPath - lockDeps[i] = d - } - } - - // Update the dependencies in kcl.mod.lock. - for _, d := range lockDeps { - kclPkg.Dependencies.Deps.Set(d.Name, d) - } - - return nil -} - -// VendorDeps will vendor all the dependencies of the current kcl package. -func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error { - // Mkdir the dir "vendor". - vendorPath := kclPkg.LocalVendorPath() - err := os.MkdirAll(vendorPath, 0755) - if err != nil { - return err - } - - return c.vendorDeps(kclPkg, vendorPath) -} - // FillDepInfo will fill registry information for a dependency. func (c *KpmClient) FillDepInfo(dep *pkg.Dependency, homepath string) error { // Homepath for a dependency is the homepath of the kcl package. @@ -1428,9 +1328,9 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc } // dependencyExists will check whether the dependency exists in the local filesystem. -func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency) (*pkg.Dependency, error) { +func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency, isVendor bool) (*pkg.Dependency, error) { // If the flag '--no_sum_check' is set, skip the checksum check. - deppath := c.getDepStorePath(searchPath, dep, false) + deppath := c.getDepStorePath(searchPath, dep, isVendor) if utils.DirExists(deppath) { depPkg, err := c.LoadPkgFromPath(deppath) if err != nil { @@ -1440,6 +1340,13 @@ func (c *KpmClient) dependencyExistsLocal(searchPath string, dep *pkg.Dependency // TODO: new local dependency structure will replace this // issue: https://github.com/kcl-lang/kpm/issues/384 dep.FullName = dep.GenDepFullName() + + if dep.GetPackage() != "" { + dep.LocalFullPath, err = utils.FindPackage(dep.LocalFullPath, dep.GetPackage()) + if err != nil { + return nil, err + } + } return dep, nil } return nil, nil @@ -1459,7 +1366,7 @@ func (c *KpmClient) DownloadDeps(deps *pkg.Dependencies, lockDeps *pkg.Dependenc return nil, errors.InvalidDependency } - existDep, err := c.dependencyExistsLocal(pkghome, &d) + existDep, err := c.dependencyExistsLocal(pkghome, &d, false) if existDep != nil && err == nil { newDeps.Deps.Set(d.Name, *existDep) continue diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 84052b98..346e859d 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -75,7 +75,6 @@ func TestWithGlobalLock(t *testing.T) { test.RunTestWithGlobalLock(t, "TestResolveMetadataInJsonStr", testResolveMetadataInJsonStr) test.RunTestWithGlobalLock(t, "testPackageCurrentPkgPath", testPackageCurrentPkgPath) test.RunTestWithGlobalLock(t, "TestUpdateKclModAndLock", testUpdateKclModAndLock) - test.RunTestWithGlobalLock(t, "TestVendorDeps", testVendorDeps) test.RunTestWithGlobalLock(t, "TestResolveDepsWithOnlyKclMod", testResolveDepsWithOnlyKclMod) test.RunTestWithGlobalLock(t, "TestResolveDepsVendorMode", testResolveDepsVendorMode) test.RunTestWithGlobalLock(t, "TestCompileWithEntryFile", testCompileWithEntryFile) @@ -528,68 +527,6 @@ func testUpdateKclModAndLock(t *testing.T) { } } -func testVendorDeps(t *testing.T) { - testDir := getTestDir("resolve_deps") - kpm_home := filepath.Join(testDir, "kpm_home") - os.RemoveAll(filepath.Join(testDir, "my_kcl")) - kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) - kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) - - depKcl1 := pkg.Dependency{ - Name: "kcl1", - FullName: "kcl1", - Version: "0.0.1", - Sum: kcl1Sum, - } - - depKcl2 := pkg.Dependency{ - Name: "kcl2", - FullName: "kcl2", - Version: "0.0.1", - Sum: kcl2Sum, - } - - mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() - mppTest.Set("kcl1", depKcl1) - mppTest.Set("kcl2", depKcl2) - - kclPkg := pkg.KclPkg{ - ModFile: pkg.ModFile{ - HomePath: filepath.Join(testDir, "my_kcl"), - // Whether the current package uses the vendor mode - // In the vendor mode, kpm will look for the package in the vendor subdirectory - // in the current package directory. - VendorMode: false, - Dependencies: pkg.Dependencies{ - Deps: mppTest, - }, - }, - HomePath: filepath.Join(testDir, "my_kcl"), - // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, - // not the dependencies in kcl.mod. - Dependencies: pkg.Dependencies{ - Deps: mppTest, - }, - } - - mykclVendorPath := filepath.Join(filepath.Join(testDir, "my_kcl"), "vendor") - assert.Equal(t, utils.DirExists(mykclVendorPath), false) - kpmcli, err := NewKpmClient() - kpmcli.homePath = kpm_home - assert.Equal(t, err, nil) - err = kpmcli.VendorDeps(&kclPkg) - assert.Equal(t, err, nil) - assert.Equal(t, utils.DirExists(mykclVendorPath), true) - assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1_0.0.1")), true) - assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2_0.0.1")), true) - - maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg) - assert.Equal(t, err, nil) - assert.Equal(t, len(maps), 2) - - os.RemoveAll(filepath.Join(testDir, "my_kcl")) -} - func testResolveDepsWithOnlyKclMod(t *testing.T) { testDir := getTestDir("resolve_dep_with_kclmod") assert.Equal(t, utils.DirExists(filepath.Join(testDir, "kcl.mod.lock")), false) @@ -622,6 +559,13 @@ func testResolveDepsVendorMode(t *testing.T) { FullName: "kcl1_0.0.1", Version: "0.0.1", Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl1", + Tag: "0.0.1", + }, + }, } depKcl2 := pkg.Dependency{ @@ -629,6 +573,13 @@ func testResolveDepsVendorMode(t *testing.T) { FullName: "kcl2_0.0.1", Version: "0.0.1", Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() @@ -687,6 +638,13 @@ func testCompileWithEntryFile(t *testing.T) { FullName: "kcl1_0.0.1", Version: "0.0.1", Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) depKcl2 := pkg.Dependency{ @@ -694,6 +652,13 @@ func testCompileWithEntryFile(t *testing.T) { FullName: "kcl2_0.0.1", Version: "0.0.1", Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, } mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() @@ -796,7 +761,7 @@ func testResolveMetadataInJsonStr(t *testing.T) { expectedDep.Deps["flask_demo_kcl_manifests"] = pkg.Dependency{ Name: "flask_demo_kcl_manifests", FullName: "flask-demo-kcl-manifests_ade147b", - Version: "ade147b", + Version: "0.1.0", LocalFullPath: filepath.Join(globalPkgPath, "flask-demo-kcl-manifests_ade147b"), } @@ -819,7 +784,7 @@ func testResolveMetadataInJsonStr(t *testing.T) { expectedDep.Deps["flask_demo_kcl_manifests"] = pkg.Dependency{ Name: "flask_demo_kcl_manifests", FullName: "flask-demo-kcl-manifests_ade147b", - Version: "ade147b", + Version: "0.1.0", LocalFullPath: filepath.Join(vendorDir, "flask-demo-kcl-manifests_ade147b"), } @@ -869,13 +834,13 @@ func testResolveMetadataInJsonStrWithPackage(t *testing.T) { Deps: make(map[string]pkg.Dependency), } - localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld") + localFullPath, err := utils.FindPackage(filepath.Join(globalPkgPath, "flask-demo-kcl-manifests_8308200"), "cc") assert.Equal(t, err, nil) - expectedDep.Deps["helloworld"] = pkg.Dependency{ - Name: "helloworld", - FullName: "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8", - Version: "ee03122b5f45b09eb48694422fc99a0772f6bba8", + expectedDep.Deps["cc"] = pkg.Dependency{ + Name: "cc", + FullName: "flask-demo-kcl-manifests_8308200", + Version: "8308200", LocalFullPath: localFullPath, } @@ -903,19 +868,19 @@ func testResolveMetadataInJsonStrWithPackage(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, utils.DirExists(vendorDir), true) - assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8")), true) + assert.Equal(t, utils.DirExists(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200")), true) - localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8"), "helloworld") + localFullPath, err = utils.FindPackage(filepath.Join(vendorDir, "flask-demo-kcl-manifests_8308200"), "cc") assert.Equal(t, err, nil) expectedDep = pkg.DependenciesUI{ Deps: make(map[string]pkg.Dependency), } - expectedDep.Deps["helloworld"] = pkg.Dependency{ - Name: "helloworld", - FullName: "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8", - Version: "ee03122b5f45b09eb48694422fc99a0772f6bba8", + expectedDep.Deps["cc"] = pkg.Dependency{ + Name: "cc", + FullName: "flask-demo-kcl-manifests_8308200", + Version: "8308200", LocalFullPath: localFullPath, } diff --git a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod index f6bc0ae4..c74af8a1 100644 --- a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod +++ b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod @@ -4,4 +4,4 @@ edition = "v0.8.0" version = "0.0.1" [dependencies] -helloworld = { git = "https://github.com/kcl-lang/modules.git", commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8", package = "helloworld" } +cc = { git = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git", commit = "8308200", package = "cc" } diff --git a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock index eb0223ed..32355765 100644 --- a/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock +++ b/pkg/client/test_data/resolve_metadata/with_package/kcl.mod.lock @@ -1,8 +1,8 @@ [dependencies] - [dependencies.helloworld] - name = "helloworld" - full_name = "modules_ee03122b5f45b09eb48694422fc99a0772f6bba8" - version = "0.1.2" - url = "https://github.com/kcl-lang/modules.git" - commit = "ee03122b5f45b09eb48694422fc99a0772f6bba8" - package = "helloworld" + [dependencies.cc] + name = "cc" + full_name = "flask-demo-kcl-manifests_8308200" + version = "0.0.1" + url = "https://github.com/kcl-lang/flask-demo-kcl-manifests.git" + commit = "8308200" + package = "cc" diff --git a/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod b/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod new file mode 100644 index 00000000..81a79664 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep1" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.1" diff --git a/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod.lock b/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod.lock new file mode 100644 index 00000000..8de6a50f --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep1/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.1" + version = "0.1.1" diff --git a/pkg/client/test_data/test_vendor_mvs/dep1/main.k b/pkg/client/test_data/test_vendor_mvs/dep1/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod b/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod new file mode 100644 index 00000000..95ee77f5 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod @@ -0,0 +1,7 @@ +[package] +name = "dep2" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +helloworld = "0.1.2" diff --git a/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod.lock b/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod.lock new file mode 100644 index 00000000..e64da0b4 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep2/kcl.mod.lock @@ -0,0 +1,5 @@ +[dependencies] + [dependencies.helloworld] + name = "helloworld" + full_name = "helloworld_0.1.2" + version = "0.1.2" diff --git a/pkg/client/test_data/test_vendor_mvs/dep2/main.k b/pkg/client/test_data/test_vendor_mvs/dep2/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/dep2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod b/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod new file mode 100644 index 00000000..b1567789 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "pkg" +edition = "v0.10.0" +version = "0.0.1" + +[dependencies] +dep1 = { path = "../dep1" } +dep2 = { path = "../dep2" } diff --git a/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod.lock b/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod.lock new file mode 100644 index 00000000..88838e6e --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/pkg/kcl.mod.lock @@ -0,0 +1,9 @@ +[dependencies] + [dependencies.dep1] + name = "dep1" + full_name = "dep1_0.0.1" + version = "0.0.1" + [dependencies.dep2] + name = "dep2" + full_name = "dep2_0.0.1" + version = "0.0.1" diff --git a/pkg/client/test_data/test_vendor_mvs/pkg/main.k b/pkg/client/test_data/test_vendor_mvs/pkg/main.k new file mode 100644 index 00000000..fa7048e6 --- /dev/null +++ b/pkg/client/test_data/test_vendor_mvs/pkg/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/pkg/client/vendor.go b/pkg/client/vendor.go new file mode 100644 index 00000000..7c026bea --- /dev/null +++ b/pkg/client/vendor.go @@ -0,0 +1,294 @@ +package client + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/elliotchance/orderedmap/v2" + "github.com/hashicorp/go-version" + "github.com/otiai10/copy" + "kcl-lang.io/kpm/pkg/constants" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/errors" + "kcl-lang.io/kpm/pkg/features" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/utils" + "kcl-lang.io/kpm/pkg/visitor" +) + +// VendorDeps will vendor all the dependencies of the current kcl package. +func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error { + // Mkdir the dir "vendor". + vendorPath := kclPkg.LocalVendorPath() + err := os.MkdirAll(vendorPath, 0755) + if err != nil { + return err + } + + return c.vendorDeps(kclPkg, vendorPath) +} + +func (c *KpmClient) vendorDeps(kclPkg *pkg.KclPkg, vendorPath string) error { + if ok, err := features.Enabled(features.SupportMVS); err == nil && ok { + // Select all the vendored dependencies + // and fill the vendored dependencies into kclPkg.Dependencies.Deps + err := c.selectVendoredDeps(kclPkg, vendorPath, kclPkg.Dependencies.Deps) + if err != nil { + return err + } + + // Move all the selected vendored dependencies to the vendor directory. + for _, depName := range kclPkg.Dependencies.Deps.Keys() { + dep, ok := kclPkg.Dependencies.Deps.Get(depName) + if !ok { + return fmt.Errorf("failed to get dependency %s", depName) + } + + // Check if the dependency is already vendored in the vendor directory. + existLocalDep, err := c.dependencyExistsLocal(filepath.Dir(vendorPath), &dep, true) + if err != nil { + return err + } + + if existLocalDep == nil { + vendorFullPath := filepath.Join(vendorPath, dep.GenDepFullName()) + cacheFullPath := filepath.Join(c.homePath, dep.GenDepFullName()) + if !utils.DirExists(vendorFullPath) { + err := copy.Copy(cacheFullPath, vendorFullPath) + if err != nil { + return err + } + } + // Load the vendored dependency + existLocalDep, err = c.dependencyExistsLocal(filepath.Dir(vendorPath), &dep, true) + if err != nil { + return err + } + + if existLocalDep == nil { + return fmt.Errorf("failed to find the vendored dependency %s", depName) + } + } + kclPkg.Dependencies.Deps.Set(depName, *existLocalDep) + } + } else { + lockDeps := make([]pkg.Dependency, 0, kclPkg.Dependencies.Deps.Len()) + for _, k := range kclPkg.Dependencies.Deps.Keys() { + d, _ := kclPkg.Dependencies.Deps.Get(k) + lockDeps = append(lockDeps, d) + } + + // Traverse all dependencies in kcl.mod.lock. + for i := 0; i < len(lockDeps); i++ { + d := lockDeps[i] + if len(d.Name) == 0 { + return errors.InvalidDependency + } + // If the dependency is from the local path, do not vendor it, vendor its dependencies. + if d.IsFromLocal() { + dpkg, err := c.LoadPkgFromPath(d.GetLocalFullPath(kclPkg.HomePath)) + if err != nil { + return err + } + err = c.vendorDeps(dpkg, vendorPath) + if err != nil { + return err + } + continue + } else { + vendorFullPath := filepath.Join(vendorPath, d.GenPathSuffix()) + + // If the package already exists in the 'vendor', do nothing. + if utils.DirExists(vendorFullPath) { + d.LocalFullPath = vendorFullPath + lockDeps[i] = d + continue + } else { + // If not in the 'vendor', check the global cache. + cacheFullPath := c.getDepStorePath(c.homePath, &d, false) + if utils.DirExists(cacheFullPath) { + // If there is, copy it into the 'vendor' directory. + err := copy.Copy(cacheFullPath, vendorFullPath) + if err != nil { + return err + } + } else { + // re-download if not. + err := c.AddDepToPkg(kclPkg, &d) + if err != nil { + return err + } + // re-vendor again with new kcl.mod and kcl.mod.lock + err = c.vendorDeps(kclPkg, vendorPath) + if err != nil { + return err + } + return nil + } + } + + if d.GetPackage() != "" { + tempVendorFullPath, err := utils.FindPackage(vendorFullPath, d.GetPackage()) + if err != nil { + return err + } + vendorFullPath = tempVendorFullPath + } + + dpkg, err := c.LoadPkgFromPath(vendorFullPath) + if err != nil { + return err + } + + // Vendor the dependencies of the current dependency. + err = c.vendorDeps(dpkg, vendorPath) + if err != nil { + return err + } + d.LocalFullPath = vendorFullPath + lockDeps[i] = d + } + } + + // Update the dependencies in kcl.mod.lock. + for _, d := range lockDeps { + kclPkg.Dependencies.Deps.Set(d.Name, d) + } + + return nil + } + return nil +} + +func (c *KpmClient) selectVendoredDeps(kpkg *pkg.KclPkg, vendorPath string, vendoredDeps *orderedmap.OrderedMap[string, pkg.Dependency]) error { + // visitorSelectorFunc selects the visitor for the source. + // For remote source, it will use the RemoteVisitor and enable the cache. + // For local source, it will use the PkgVisitor. + visitorSelectorFunc := func(source *downloader.Source) (visitor.Visitor, error) { + pkgVisitor := &visitor.PkgVisitor{ + Settings: &c.settings, + LogWriter: c.logWriter, + } + + if source.IsRemote() { + return &visitor.RemoteVisitor{ + PkgVisitor: pkgVisitor, + Downloader: c.DepDownloader, + InsecureSkipTLSverify: c.insecureSkipTLSverify, + EnableCache: true, + CachePath: c.homePath, + }, nil + } else if source.IsLocalTarPath() || source.IsLocalTgzPath() { + return visitor.NewArchiveVisitor(pkgVisitor), nil + } else if source.IsLocalPath() { + rootPath, err := source.FindRootPath() + if err != nil { + return nil, err + } + kclmodpath := filepath.Join(rootPath, constants.KCL_MOD) + if utils.DirExists(kclmodpath) { + return pkgVisitor, nil + } else { + return visitor.NewVirtualPkgVisitor(pkgVisitor), nil + } + } else { + return nil, fmt.Errorf("unsupported source") + } + } + + // Iterate all the dependencies of the package in kcl.mod. + for _, depName := range kpkg.ModFile.Dependencies.Deps.Keys() { + dep, ok := kpkg.ModFile.Dependencies.Deps.Get(depName) + if !ok { + return fmt.Errorf("failed to get dependency %s", depName) + } + + // Select the dependency with the MVS + // Keep the greater version in dependencies graph + selectedDep := &dep + if existsDep, exists := vendoredDeps.Get(depName); exists && len(existsDep.Version) > 0 && + len(dep.Version) > 0 && + // TODO: Skip the git dependencies for now and get the version from the cache when the new local storage structure is complete + // the new local storage structure: https://github.com/kcl-lang/kpm/issues/384 + dep.Source.Git == nil && + existsDep.Source.Git == nil { + existsVersion, err := version.NewVersion(existsDep.Version) + if err != nil { + return err + } + depVersion, err := version.NewVersion(dep.Version) + if err != nil { + return err + } + // Select the greater version + if existsVersion.GreaterThan(depVersion) { + selectedDep = &existsDep + } + } + + // Check if the dependency is already vendored in the vendor directory. + existLocalDep, err := c.dependencyExistsLocal(filepath.Dir(vendorPath), selectedDep, true) + if err != nil { + return err + } + + // If the dependency is already vendored, just update the dependency path. + if existLocalDep != nil { + // Collect the vendored dependency + vendoredDeps.Set(depName, *existLocalDep) + // Load the vendored dependency + dpkg, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(existLocalDep.LocalFullPath), + pkg.WithSettings(&c.settings), + ) + if err != nil { + return err + } + // Vendor the indirected dependencies of the vendored dependency + err = c.selectVendoredDeps(dpkg, vendorPath, vendoredDeps) + if err != nil { + return err + } + } else { + // If the dependency is not vendored in the vendor directory + selectDepSource := &selectedDep.Source + // Check if the dependency is a local path and it is not an absolute path. + // If it is not an absolute path, transform the path to an absolute path. + if selectDepSource.IsLocalPath() && !filepath.IsAbs(selectDepSource.Local.Path) { + selectDepSource = &downloader.Source{ + Local: &downloader.Local{ + Path: filepath.Join(kpkg.HomePath, selectDepSource.Local.Path), + }, + } + } + + // By visitor, if the dependency is a remote source, it will download and load the dependency + // if the dependency is a local source, it will load the dependency. + // if the dependency is cached, it will load the dependency from the cache. + pkgVisitor, err := visitorSelectorFunc(selectDepSource) + if err != nil { + return err + } + err = pkgVisitor.Visit(selectDepSource, + func(kclPkg *pkg.KclPkg) error { + existLocalDep, err := c.dependencyExistsLocal(c.homePath, selectedDep, false) + if err != nil { + return err + } + + if existLocalDep == nil { + return fmt.Errorf("failed to find the vendored dependency %s", depName) + } + // Collect the vendored dependency + vendoredDeps.Set(depName, *existLocalDep) + return nil + }, + ) + if err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/client/vendor_test.go b/pkg/client/vendor_test.go new file mode 100644 index 00000000..db388c0b --- /dev/null +++ b/pkg/client/vendor_test.go @@ -0,0 +1,118 @@ +package client + +import ( + "os" + "path/filepath" + "testing" + + "github.com/elliotchance/orderedmap/v2" + "github.com/stretchr/testify/assert" + "kcl-lang.io/kpm/pkg/downloader" + "kcl-lang.io/kpm/pkg/features" + pkg "kcl-lang.io/kpm/pkg/package" + "kcl-lang.io/kpm/pkg/settings" + "kcl-lang.io/kpm/pkg/test" + "kcl-lang.io/kpm/pkg/utils" +) + +func testVendorDeps(t *testing.T) { + testDir := getTestDir("resolve_deps") + kpm_home := filepath.Join(testDir, "kpm_home") + os.RemoveAll(filepath.Join(testDir, "my_kcl")) + kcl1Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl1")) + kcl2Sum, _ := utils.HashDir(filepath.Join(kpm_home, "kcl2")) + + depKcl1 := pkg.Dependency{ + Name: "kcl1", + FullName: "kcl1", + Version: "0.0.1", + Sum: kcl1Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl1", + Tag: "0.0.1", + }, + }, + } + + depKcl2 := pkg.Dependency{ + Name: "kcl2", + FullName: "kcl2", + Version: "0.0.1", + Sum: kcl2Sum, + Source: downloader.Source{ + Oci: &downloader.Oci{ + Reg: "ghcr.io", + Repo: "kcl-lang/kcl2", + Tag: "0.0.1", + }, + }, + } + + mppTest := orderedmap.NewOrderedMap[string, pkg.Dependency]() + mppTest.Set("kcl1", depKcl1) + mppTest.Set("kcl2", depKcl2) + + kclPkg := pkg.KclPkg{ + ModFile: pkg.ModFile{ + HomePath: filepath.Join(testDir, "my_kcl"), + // Whether the current package uses the vendor mode + // In the vendor mode, kpm will look for the package in the vendor subdirectory + // in the current package directory. + VendorMode: false, + Dependencies: pkg.Dependencies{ + Deps: mppTest, + }, + }, + HomePath: filepath.Join(testDir, "my_kcl"), + // The dependencies in the current kcl package are the dependencies of kcl.mod.lock, + // not the dependencies in kcl.mod. + Dependencies: pkg.Dependencies{ + Deps: mppTest, + }, + } + + mykclVendorPath := filepath.Join(filepath.Join(testDir, "my_kcl"), "vendor") + assert.Equal(t, utils.DirExists(mykclVendorPath), false) + kpmcli, err := NewKpmClient() + kpmcli.homePath = kpm_home + assert.Equal(t, err, nil) + err = kpmcli.VendorDeps(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, utils.DirExists(mykclVendorPath), true) + assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl1_0.0.1")), true) + assert.Equal(t, utils.DirExists(filepath.Join(mykclVendorPath, "kcl2_0.0.1")), true) + + maps, err := kpmcli.ResolveDepsIntoMap(&kclPkg) + assert.Equal(t, err, nil) + assert.Equal(t, len(maps), 2) + + os.RemoveAll(filepath.Join(testDir, "my_kcl")) +} + +func testVendorWithMVS(t *testing.T) { + features.Enable(features.SupportMVS) + defer features.Disable(features.SupportMVS) + testDir := getTestDir("test_vendor_mvs") + pkgPath := filepath.Join(testDir, "pkg") + kPkg, err := pkg.LoadKclPkgWithOpts( + pkg.WithPath(pkgPath), + pkg.WithSettings(settings.GetSettings()), + ) + assert.Equal(t, err, nil) + + kpmcli, err := NewKpmClient() + assert.Equal(t, err, nil) + err = kpmcli.VendorDeps(kPkg) + assert.Equal(t, err, nil) + + assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "vendor")), true) + assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "vendor", "helloworld_0.1.2")), true) + assert.Equal(t, utils.DirExists(filepath.Join(pkgPath, "vendor", "helloworld_0.1.1")), false) +} + +func TestVendorWithGlobalLock(t *testing.T) { + test.RunTestWithGlobalLock(t, "TestVendorDeps", testVendorDeps) + test.RunTestWithGlobalLock(t, "TestVendorWithMVS", testVendorWithMVS) +} diff --git a/test/e2e/test_suites/test_data/kcl1.tar b/test/e2e/test_suites/test_data/kcl1.tar index 37427392..75991074 100644 Binary files a/test/e2e/test_suites/test_data/kcl1.tar and b/test/e2e/test_suites/test_data/kcl1.tar differ