Skip to content

Commit

Permalink
Add trace logging to Kitfile generation
Browse files Browse the repository at this point in the history
  • Loading branch information
@amisevsk
amisevsk committed Dec 18, 2024
1 parent 0359443 commit 7fbd3d6
Showing 1 changed file with 31 additions and 9 deletions.
40 changes: 31 additions & 9 deletions pkg/lib/kitfile/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,13 +70,15 @@ var datasetSuffixes = []string{
// packageOpt can be used to define metadata for the Kitfile (i.e. the package
// section), which is left empty if the parameter is nil.
func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.KitFile, error) {
output.Logf(output.LogLevelTrace, "Generating Kitfile in %s", baseDir)
kitfile := &artifact.KitFile{
ManifestVersion: "1.0.0",
}
if packageOpt != nil {
kitfile.Package = *packageOpt
}

output.Logf(output.LogLevelTrace, "Reading directory contents")
ds, err := os.ReadDir(baseDir)
if err != nil {
return nil, fmt.Errorf("error reading directory: %w", err)
Expand All @@ -93,6 +95,7 @@ func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.Ki
for _, d := range ds {
filename := d.Name()
if constants.IsDefaultKitfileName(filename) {
output.Logf(output.LogLevelTrace, "Skipping Kitfile '%s'", filename)
// Skip Kitfile files (if present in the directory...). These won't be packed
// either way.
continue
Expand All @@ -108,12 +111,14 @@ func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.Ki

// Check for "special" files (e.g. readme, license)
if strings.HasPrefix(strings.ToLower(filename), "readme") {
output.Logf(output.LogLevelTrace, "Found readme file '%s'", filename)
kitfile.Docs = append(kitfile.Docs, artifact.Docs{
Path: filename,
Description: "Readme file",
})
continue
} else if strings.HasPrefix(strings.ToLower(filename), "license") {
output.Logf(output.LogLevelTrace, "Found license file '%s'", filename)
kitfile.Docs = append(kitfile.Docs, artifact.Docs{
Path: filename,
Description: "License file",
Expand All @@ -123,6 +128,7 @@ func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.Ki
output.Debugf("Error determining license type: %s", err)
output.Logf(output.LogLevelWarn, "Unable to determine license type")
}
output.Logf(output.LogLevelTrace, "Detected license %s for license file", detectedLicenseType)
detectedLicenseType = licenseType
continue
}
Expand All @@ -136,31 +142,39 @@ func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.Ki
case fileTypeMetadata:
// Metadata should be included in either Model or Datasets, depending on
// other contents
output.Logf(output.LogLevelTrace, "Detected metadata file '%s'", filename)
metadataPaths = append(metadataPaths, filename)
case fileTypeDocs:
kitfile.Docs = append(kitfile.Docs, artifact.Docs{Path: filename})
case fileTypeDataset:
kitfile.DataSets = append(kitfile.DataSets, artifact.DataSet{Path: filename})
default:
output.Logf(output.LogLevelTrace, "File %s is either code or unknown type. Will be added as a catch-all section", filename)
// File is either code or unknown; we'll have to include it in a catch-all section
includeCatchallSection = true
}
}

if len(modelFiles) > 0 {
addModelToKitfile(kitfile, baseDir, modelFiles)
if err := addModelToKitfile(kitfile, baseDir, modelFiles); err != nil {
return nil, fmt.Errorf("failed to add model to Kitfile: %w", err)
}
output.Logf(output.LogLevelTrace, "Adding metadata files as model parts")
for _, metadataPath := range metadataPaths {
kitfile.Model.Parts = append(kitfile.Model.Parts, artifact.ModelPart{Path: metadataPath})
}
} else {
output.Logf(output.LogLevelTrace, "No model detected; adding metadata files as dataset layers")
for _, metadataPath := range metadataPaths {
kitfile.DataSets = append(kitfile.DataSets, artifact.DataSet{Path: metadataPath})
}
}

// Decide how to handle remaining paths. Either package them in one large code layer with basePath
// or as separate layers for each directory.
output.Logf(output.LogLevelTrace, "Unable to process %d paths in %s", len(unprocessedDirPaths), baseDir)
if includeCatchallSection || len(unprocessedDirPaths) > 5 {
output.Logf(output.LogLevelTrace, "Adding catch-all code layer to include files in %s", baseDir)
// Overwrite any code layers we added before; this is cleaner than e.g. having a layer for '.' and a layer for 'src'
kitfile.Code = []artifact.Code{{Path: "."}}
} else {
Expand All @@ -177,37 +191,40 @@ func GenerateKitfile(baseDir string, packageOpt *artifact.Package) (*artifact.Ki
} else if len(kitfile.Code) == 1 {
kitfile.Code[0].License = detectedLicenseType
} else {
output.Logf(output.LogLevelTrace, "Unsure what license applies to, adding to Kitfile package")
kitfile.Package.License = detectedLicenseType
}

return kitfile, nil
}

func addDirToKitfile(kitfile *artifact.KitFile, path string, d fs.DirEntry) (modelFiles []string, err error) {
func addDirToKitfile(kitfile *artifact.KitFile, dirPath string, d fs.DirEntry) (modelFiles []string, err error) {
switch d.Name() {
case "docs":
output.Logf(output.LogLevelTrace, "Directory %s interpreted as documentation", d.Name())
kitfile.Docs = append(kitfile.Docs, artifact.Docs{
Path: path,
Path: dirPath,
})
return nil, nil
case "src", "pkg", "lib", "build":
output.Logf(output.LogLevelTrace, "Directory %s interpreted as code", d.Name())
kitfile.Code = append(kitfile.Code, artifact.Code{
Path: path,
Path: dirPath,
})
return nil, nil
}

entries, err := os.ReadDir(path)
entries, err := os.ReadDir(dirPath)
if err != nil {
return nil, fmt.Errorf("failed to read directory %s: %w", path, err)
return nil, fmt.Errorf("failed to read directory %s: %w", dirPath, err)
}

// Sort entries in the directory to try and figure out what it contains. We'll reuse the
// fact that the fileTypes are enumerated using iota (and so are ints) to index correctly.
// Avoid using maps here since they iterate in a random order.
directoryContents := [int(fileTypeUnknown) + 1][]string{}
for _, entry := range entries {
relPath := filepath.Join(path, entry.Name())
relPath := filepath.Join(dirPath, entry.Name())
if entry.IsDir() {
// TODO: we can potentially recurse further here if we find we need to
directoryContents[int(fileTypeUnknown)] = append(directoryContents[int(fileTypeUnknown)], relPath)
Expand All @@ -226,6 +243,7 @@ func addDirToKitfile(kitfile *artifact.KitFile, path string, d fs.DirEntry) (mod
for fType, files := range directoryContents {
if len(files) > 0 && fileType(fType) != fileTypeMetadata {
if overallFiletype != fileTypeUnknown {
output.Logf(output.LogLevelTrace, "Detected mixed contents within directory %s", dirPath)
directoryHasMixedContents = true
}
overallFiletype = fileType(fType)
Expand All @@ -236,13 +254,17 @@ func addDirToKitfile(kitfile *artifact.KitFile, path string, d fs.DirEntry) (mod
}
switch overallFiletype {
case fileTypeModel:
output.Logf(output.LogLevelTrace, "Interpreting directory %s as a model directory", dirPath)
// Include any metadata files as modelParts later
modelFiles = append(modelFiles, directoryContents[int(fileTypeMetadata)]...)
case fileTypeDataset:
kitfile.DataSets = append(kitfile.DataSets, artifact.DataSet{Path: path})
output.Logf(output.LogLevelTrace, "Interpreting directory %s as a dataset directory", dirPath)
kitfile.DataSets = append(kitfile.DataSets, artifact.DataSet{Path: dirPath})
case fileTypeDocs:
kitfile.Docs = append(kitfile.Docs, artifact.Docs{Path: path})
output.Logf(output.LogLevelTrace, "Interpreting directory %s as a docs directory", dirPath)
kitfile.Docs = append(kitfile.Docs, artifact.Docs{Path: dirPath})
default:
output.Logf(output.LogLevelTrace, "Could not determine type for directory %s", dirPath)
// If it's overall code, metadata, or unknown, just return it as unprocessed and let it be added as a Code section
// later
return modelFiles, fmt.Errorf("directory should be handled as Code")
Expand Down

0 comments on commit 7fbd3d6

Please sign in to comment.