Skip to content

Commit

Permalink
Merge pull request #523 from kcl-lang/fix-missing-mod-spec
Browse files Browse the repository at this point in the history 
fix: fix missing mod and deps not found
  • Loading branch information
@Peefy
Peefy authored Nov 7, 2024
2 parents bc053bc + 657392d commit 735bff3
Show file tree
Hide file tree
Showing 5 changed files with 155 additions and 84 deletions.
153 changes: 72 additions & 81 deletions pkg/client/add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,54 +13,46 @@ import (
type AddOptions struct {
// Source is the source of the package to be pulled.
// Including git, oci, local.
Sources []*downloader.Source
KclPkg *pkg.KclPkg
Source *downloader.Source
KclPkg *pkg.KclPkg
}

type AddOption func(*AddOptions) error

func WithAddSource(source *downloader.Source) AddOption {
func WithAddModSpec(modSpec *downloader.ModSpec) AddOption {
return func(opts *AddOptions) error {
if opts.Sources == nil {
opts.Sources = make([]*downloader.Source, 0)
if modSpec == nil {
return fmt.Errorf("modSpec cannot be nil")
}
opts.Sources = append(opts.Sources, source)
if opts.Source == nil {
opts.Source = &downloader.Source{
ModSpec: modSpec,
}
} else {
opts.Source.ModSpec = modSpec
}

return nil
}
}

func WithAddSources(sources []*downloader.Source) AddOption {
return func(ro *AddOptions) error {
ro.Sources = sources
func WithAddSource(source *downloader.Source) AddOption {
return func(opts *AddOptions) error {
if source == nil {
return fmt.Errorf("source cannot be nil")
}
opts.Source = source
return nil
}
}

func WithAddSourceUrl(sourceUrl string) AddOption {
return func(opts *AddOptions) error {
if opts.Sources == nil {
opts.Sources = make([]*downloader.Source, 0)
}
source, err := downloader.NewSourceFromStr(sourceUrl)
if err != nil {
return err
}
opts.Sources = append(opts.Sources, source)
return nil
}
}

func WithAddSourceUrls(sourceUrls []string) AddOption {
return func(opts *AddOptions) error {
var sources []*downloader.Source
for _, sourceUrl := range sourceUrls {
source, err := downloader.NewSourceFromStr(sourceUrl)
if err != nil {
return err
}
sources = append(sources, source)
}
opts.Sources = sources
opts.Source = source
return nil
}
}
Expand Down Expand Up @@ -88,6 +80,7 @@ func (c *KpmClient) Add(options ...AddOption) error {
}
}
addedPkg := opts.KclPkg
depSource := opts.Source

visitorSelector := func(source *downloader.Source) (visitor.Visitor, error) {
pkgVisitor := &visitor.PkgVisitor{
Expand All @@ -113,73 +106,71 @@ func (c *KpmClient) Add(options ...AddOption) error {
}
}

for _, depSource := range opts.Sources {
// Set the default OCI registry and repo if the source is nil and the package spec is not nil.
if depSource.IsNilSource() && !depSource.ModSpec.IsNil() {
depSource.Oci = &downloader.Oci{
Reg: c.GetSettings().Conf.DefaultOciRegistry,
Repo: utils.JoinPath(c.GetSettings().Conf.DefaultOciRepo, depSource.ModSpec.Name),
Tag: depSource.ModSpec.Version,
}
// Set the default OCI registry and repo if the source is nil and the package spec is not nil.
if depSource.IsNilSource() && !depSource.ModSpec.IsNil() {
depSource.Oci = &downloader.Oci{
Reg: c.GetSettings().Conf.DefaultOciRegistry,
Repo: utils.JoinPath(c.GetSettings().Conf.DefaultOciRepo, depSource.ModSpec.Name),
Tag: depSource.ModSpec.Version,
}
}

var fullSouce *downloader.Source
// Transform the relative path to the full path.
if depSource.IsLocalPath() && !filepath.IsAbs(depSource.Path) {
fullSouce = &downloader.Source{
ModSpec: depSource.ModSpec,
Local: &downloader.Local{
Path: filepath.Join(addedPkg.HomePath, depSource.Path),
},
}
} else {
fullSouce = depSource
var fullSouce *downloader.Source
// Transform the relative path to the full path.
if depSource.IsLocalPath() && !filepath.IsAbs(depSource.Path) {
fullSouce = &downloader.Source{
ModSpec: depSource.ModSpec,
Local: &downloader.Local{
Path: filepath.Join(addedPkg.HomePath, depSource.Path),
},
}
} else {
fullSouce = depSource
}

visitor, err := visitorSelector(fullSouce)
if err != nil {
return err
}
visitor, err := visitorSelector(fullSouce)
if err != nil {
return err
}

// Visit the dependency source
// If the dependency is remote, the visitor will download it to the local.
// If the dependency is already in local cache, the visitor will not download it again.
err = visitor.Visit(fullSouce, func(depPkg *pkg.KclPkg) error {
var modSpec *downloader.ModSpec
if depSource.ModSpec.IsNil() {
modSpec = &downloader.ModSpec{
Name: depPkg.ModFile.Pkg.Name,
Version: depPkg.ModFile.Pkg.Version,
}
depSource.ModSpec = modSpec
// Visit the dependency source
// If the dependency is remote, the visitor will download it to the local.
// If the dependency is already in local cache, the visitor will not download it again.
err = visitor.Visit(fullSouce, func(depPkg *pkg.KclPkg) error {
var modSpec *downloader.ModSpec
if depSource.ModSpec.IsNil() {
modSpec = &downloader.ModSpec{
Name: depPkg.ModFile.Pkg.Name,
Version: depPkg.ModFile.Pkg.Version,
}
depSource.ModSpec = modSpec
}

dep := pkg.Dependency{
Name: depPkg.ModFile.Pkg.Name,
FullName: depPkg.GetPkgFullName(),
Version: depPkg.ModFile.Pkg.Version,
LocalFullPath: depPkg.HomePath,
Source: *depSource,
}
dep := pkg.Dependency{
Name: depPkg.ModFile.Pkg.Name,
FullName: depPkg.GetPkgFullName(),
Version: depPkg.ModFile.Pkg.Version,
LocalFullPath: depPkg.HomePath,
Source: *depSource,
}

// Add the dependency to the kcl.mod file.
if modExistDep, ok := addedPkg.ModFile.Dependencies.Deps.Get(dep.Name); ok {
if less, err := modExistDep.VersionLessThan(&dep); less && err == nil {
addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep)
}
} else {
// Add the dependency to the kcl.mod file.
if modExistDep, ok := addedPkg.ModFile.Dependencies.Deps.Get(dep.Name); ok {
if less, err := modExistDep.VersionLessThan(&dep); less && err == nil {
addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep)
}

return nil
})
if err != nil {
return err
} else {
addedPkg.ModFile.Dependencies.Deps.Set(dep.Name, dep)
}

return nil
})
if err != nil {
return err
}

// Iterate the dependencies and update the kcl.mod and kcl.mod.lock respectively.
_, err := c.Update(
_, err = c.Update(
WithUpdatedKclPkg(addedPkg),
)

Expand Down
18 changes: 18 additions & 0 deletions pkg/client/pull.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,23 @@ type PullOptions struct {

type PullOption func(*PullOptions) error

func WithPullModSpec(modSpec *downloader.ModSpec) PullOption {
return func(opts *PullOptions) error {
if modSpec == nil {
return errors.New("modSpec cannot be nil")
}
if opts.Source == nil {
opts.Source = &downloader.Source{
ModSpec: modSpec,
}
} else {
opts.Source.ModSpec = modSpec
}

return nil
}
}

func WithPullSourceUrl(sourceUrl string) PullOption {
return func(opts *PullOptions) error {
source, err := downloader.NewSourceFromStr(sourceUrl)
Expand Down Expand Up @@ -76,6 +93,7 @@ func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) {
if err != nil {
return nil, err
}

reporter.ReportMsgTo(
fmt.Sprintf("start to pull %s", sourceStr),
c.GetLogWriter(),
Expand Down
38 changes: 38 additions & 0 deletions pkg/client/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,29 @@ type RunOptions struct {

type RunOption func(*RunOptions) error

func WithRunModSpec(modSpec *downloader.ModSpec) RunOption {
return func(ro *RunOptions) error {
if modSpec == nil {
return errors.New("modSpec cannot be nil")
}
if ro.Sources == nil {
ro.Sources = make([]*downloader.Source, 0)
}
if len(ro.Sources) > 1 {
return errors.New("only allows one package to be compiled at a time")
}
if len(ro.Sources) == 0 {
ro.Sources = append(ro.Sources, &downloader.Source{
ModSpec: modSpec,
})
} else {
ro.Sources[0].ModSpec = modSpec
}

return nil
}
}

// Use the another RunOptions to override the current RunOptions.
func WithRunOptions(runOpts *RunOptions) RunOption {
return func(ro *RunOptions) error {
Expand Down Expand Up @@ -401,6 +424,16 @@ func (o *RunOptions) applyCompileOptions(source downloader.Source, kclPkg *pkg.K
sourcePath = filepath.Join(workDir, sourcePath)
}

// When determining whether the path in 'Source' is the same as homepath of 'KclPkg',
// use the subdirectories specified by 'ModSpec'
var err error
if pkgSource.ModSpec != nil && pkgSource.ModSpec.Name != "" {
sourcePath, err = utils.FindPackage(sourcePath, pkgSource.ModSpec.Name)
if err != nil {
return false
}
}

pkgHome := kclPkg.HomePath

if !filepath.IsAbs(pkgHome) && !utils.IsModRelativePath(sourcePath) {
Expand Down Expand Up @@ -479,6 +512,7 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) {
workDir := o.WorkDir

var pkgSource *downloader.Source
var modSpec *downloader.ModSpec
if len(o.Sources) == 0 {
workDir, err := filepath.Abs(workDir)
if err != nil {
Expand All @@ -497,6 +531,9 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) {
for _, source := range o.Sources {
if pkgSource == nil {
pkgSource = source
// If the root source is found,
// update the modSpec and rootPath.
modSpec = source.ModSpec
rootPath, err = source.FindRootPath()
if err != nil {
return nil, err
Expand Down Expand Up @@ -548,6 +585,7 @@ func (o *RunOptions) getPkgSource() (*downloader.Source, error) {
if err != nil {
return nil, err
}
pkgSource.ModSpec = modSpec
}
}

Expand Down
8 changes: 6 additions & 2 deletions pkg/downloader/source.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,7 +443,10 @@ func (source *Source) FromString(sourceStr string) error {
} else if sourceUrl.Scheme == constants.DefaultOciScheme {
source.ModSpec = &ModSpec{}
source.ModSpec.FromString(sourceUrl.String())
} else {
// There is a case where there is only 'ModSpec'
// On winodws, the path url will be parsed as 'Opaque'
// On linux, the path url will be parsed as 'Path'
} else if sourceUrl.Path != "" || sourceUrl.Opaque != "" {
source.Local = &Local{}
source.Local.FromString(sourceUrl.String())
}
Expand Down Expand Up @@ -556,7 +559,8 @@ func (git *Git) GetValidGitReference() (string, error) {
nonEmptyRef = git.Branch
}

if nonEmptyFields != 1 {
// The latest commit id for git repo is supported.
if nonEmptyFields > 1 {
return "", errors.New("only one of branch, tag or commit is allowed")
}

Expand Down
22 changes: 21 additions & 1 deletion pkg/visitor/visitor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func (pv *PkgVisitor) Visit(s *downloader.Source, v visitFunc) error {
return err
}

if !s.ModSpec.IsNil() {
if s.ModSpec != nil && s.ModSpec.Name != "" {
modPath, err = utils.FindPackage(modPath, s.ModSpec.Name)
if err != nil {
return err
Expand Down Expand Up @@ -120,6 +120,15 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error {
return fmt.Errorf("source is not remote")
}

// For some sources with only the spec, the default registry and repo will be used.
if s.SpecOnly() {
s.Oci = &downloader.Oci{
Reg: rv.Settings.DefaultOciRegistry(),
Repo: utils.JoinPath(rv.Settings.DefaultOciRepo(), s.ModSpec.Name),
Tag: s.ModSpec.Version,
}
}

var cacheFullPath string
var modFullPath string

Expand All @@ -134,6 +143,17 @@ func (rv *RemoteVisitor) Visit(s *downloader.Source, v visitFunc) error {
}
}

// If the cache is not enabled,
// create a temporary directory to get the latest commit of git repo
if !rv.EnableCache {
cacheFullPath, err = os.MkdirTemp("", "")
if err != nil {
return err
}

defer os.RemoveAll(cacheFullPath)
}

// 1. Load the credential file.
credCli, err := downloader.LoadCredentialFile(rv.Settings.CredentialsFile)
if err != nil {
Expand Down

0 comments on commit 735bff3

Please sign in to comment.