Skip to content

Commit

Permalink
refactor: input file expand function impl
Browse files Browse the repository at this point in the history 
Signed-off-by: peefy <[email protected]>
  • Loading branch information
@Peefy
Peefy committed Sep 13, 2024
1 parent ded5e87 commit d61eeb2
Show file tree
Hide file tree
Showing 30 changed files with 208 additions and 258 deletions.
2 changes: 1 addition & 1 deletion kclvm/Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions kclvm/api/src/service/service_impl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,11 @@ use std::string::String;

use crate::gpyrpc::*;

use anyhow::anyhow;
use kcl_language_server::rename;
use kclvm_config::settings::build_settings_pathbuf;
use kclvm_driver::canonicalize_input_files;
use kclvm_loader::option::list_options;
use kclvm_loader::{load_packages_with_cache, LoadPackageOptions};
use kclvm_parser::entry::get_normalized_k_files_from_paths;
use kclvm_parser::load_program;
use kclvm_parser::parse_file;
use kclvm_parser::KCLModuleCache;
Expand Down Expand Up @@ -861,12 +860,13 @@ impl KclvmServiceImpl {
let settings_files = args.files.iter().map(|f| f.as_str()).collect::<Vec<&str>>();
let settings_pathbuf = build_settings_pathbuf(&[], Some(settings_files), None)?;
let files = if !settings_pathbuf.settings().input().is_empty() {
canonicalize_input_files(
get_normalized_k_files_from_paths(
&settings_pathbuf.settings().input(),
args.work_dir.clone(),
false,
)
.map_err(|e| anyhow!(e))?
&LoadProgramOptions {
work_dir: args.work_dir.clone(),
..Default::default()
},
)?
} else {
vec![]
};
Expand Down
3 changes: 1 addition & 2 deletions kclvm/driver/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@ kclvm-utils ={ path = "../utils"}
kclvm-parser ={ path = "../parser"}
kclvm-ast ={ path = "../ast"}
walkdir = "2"

serde = { version = "1.0", features = ["derive"] }
anyhow = { version = "1.0.70", features = ["backtrace"] }
glob = "0.3.1"

flate2 = "1.0.30"
tar = "0.4.40"
indexmap = "2.2.6"
Expand Down
127 changes: 8 additions & 119 deletions kclvm/driver/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,11 @@ pub mod toolchain;
mod tests;

use anyhow::Result;
use glob::glob;
use kclvm_config::{
modfile::{
get_pkg_root, load_mod_file, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE,
KCL_MOD_PATH_ENV, KCL_WORK_FILE,
KCL_WORK_FILE,
},
path::ModRelativePath,
settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE},
workfile::load_work_file,
};
Expand All @@ -30,106 +28,6 @@ use std::{
use toolchain::{fill_pkg_maps_for_k_file, Metadata, Toolchain};
use walkdir::WalkDir;

/// Expand the single file pattern to a list of files.
pub fn expand_if_file_pattern(file_pattern: String) -> Result<Vec<String>, String> {
let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?;
let mut matched_files = vec![];

for path in paths.flatten() {
matched_files.push(path.to_string_lossy().to_string());
}

Ok(matched_files)
}

/// Expand input kcl files with the file patterns.
pub fn expand_input_files(k_files: &[String]) -> Vec<String> {
let mut res = vec![];
for file in k_files {
if let Ok(files) = expand_if_file_pattern(file.to_string()) {
if !files.is_empty() {
res.extend(files);
} else {
res.push(file.to_string())
}
} else {
res.push(file.to_string())
}
}
res
}

/// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path.
pub fn canonicalize_input_files(
k_files: &[String],
work_dir: String,
check_exist: bool,
) -> Result<Vec<String>, String> {
let mut kcl_paths = Vec::<String>::new();
// The first traversal changes the relative path to an absolute path
for file in k_files.iter() {
let path = Path::new(file);

let is_absolute = path.is_absolute();
let is_exist_maybe_symlink = path.exists();
// If the input file or path is a relative path and it is not a absolute path in the KCL module VFS,
// join with the work directory path and convert it to a absolute path.
let path = ModRelativePath::from(file.to_string());
let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? {
let filepath = Path::new(&work_dir).join(file);
match filepath.canonicalize() {
Ok(path) => Some(path.adjust_canonicalization()),
Err(_) => {
if check_exist {
return Err(format!(
"Cannot find the kcl file, please check the file path {}",
file
));
}
Some(filepath.to_string_lossy().to_string())
}
}
} else {
None
};
// If the input file or path is a symlink, convert it to a real path.
let real_path = if is_exist_maybe_symlink {
match PathBuf::from(file.to_string()).canonicalize() {
Ok(real_path) => Some(String::from(real_path.to_str().unwrap())),
Err(_) => {
if check_exist {
return Err(format!(
"Cannot find the kcl file, please check the file path {}",
file
));
}
Some(file.to_string())
}
}
} else {
None
};

kcl_paths.push(abs_path.unwrap_or(real_path.unwrap_or(file.to_string())));
}

// Get the root path of the project
let pkgroot = kclvm_config::modfile::get_pkg_root_from_paths(&kcl_paths, work_dir)?;

// The second traversal replaces ${KCL_MOD} with the project root path
kcl_paths = kcl_paths
.iter()
.map(|file| {
if file.contains(KCL_MOD_PATH_ENV) {
file.replace(KCL_MOD_PATH_ENV, pkgroot.as_str())
} else {
file.clone()
}
})
.collect();
Ok(kcl_paths)
}

/// Get compile workspace(files and options) from a single file input.
/// 1. Lookup entry files in kcl.yaml
/// 2. Lookup entry files in kcl.mod
Expand Down Expand Up @@ -183,15 +81,12 @@ pub fn lookup_compile_workspace(
work_dir: work_dir.clone(),
..Default::default()
};
match canonicalize_input_files(&files, work_dir, true) {
Ok(kcl_paths) => {
// 1. find the kcl.mod path
let metadata =
fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt)
.unwrap_or(None);
(kcl_paths, Some(load_opt), metadata)
}
Err(_) => default_res,
let metadata =
fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt).unwrap_or(None);
if files.is_empty() {
default_res
} else {
(files, Some(load_opt), metadata)
}
}
Err(_) => default_res,
Expand All @@ -205,10 +100,7 @@ pub fn lookup_compile_workspace(
if let Some(files) = mod_file.get_entries() {
let work_dir = dir.to_string_lossy().to_string();
load_opt.work_dir = work_dir.clone();
match canonicalize_input_files(&files, work_dir, true) {
Ok(kcl_paths) => (kcl_paths, Some(load_opt), metadata),
Err(_) => default_res,
}
(files, Some(load_opt), metadata)
} else {
default_res
}
Expand Down Expand Up @@ -267,7 +159,6 @@ pub fn lookup_compile_workspaces(
(vec![path.to_string()], Some(load_opt), metadata),
);
}

WorkSpaceKind::SettingFile(setting_file) => {
workspaces.insert(
workspace.clone(),
Expand All @@ -278,7 +169,6 @@ pub fn lookup_compile_workspaces(
),
);
}

WorkSpaceKind::ModFile(mod_file) => {
workspaces.insert(
workspace.clone(),
Expand All @@ -289,7 +179,6 @@ pub fn lookup_compile_workspaces(
),
);
}

WorkSpaceKind::File(_) | WorkSpaceKind::NotFound => {
let pathbuf = PathBuf::from(path);
let file_path = pathbuf.as_path();
Expand Down
102 changes: 2 additions & 100 deletions kclvm/driver/src/tests.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::{env, fs, panic};

use kclvm_config::modfile::get_vendor_home;
Expand All @@ -9,105 +9,7 @@ use walkdir::WalkDir;
use crate::arguments::parse_key_value_pair;
use crate::toolchain::Toolchain;
use crate::toolchain::{fill_pkg_maps_for_k_file, CommandToolchain, NativeToolchain};
use crate::{canonicalize_input_files, expand_input_files, get_pkg_list};
use crate::{lookup_the_nearest_file_dir, toolchain};

#[test]
fn test_canonicalize_input_files() {
let input_files = vec!["file1.k".to_string(), "file2.k".to_string()];
let work_dir = ".".to_string();
let expected_files = vec![
Path::new(".").join("file1.k").to_string_lossy().to_string(),
Path::new(".").join("file2.k").to_string_lossy().to_string(),
];
assert_eq!(
canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(),
expected_files
);
assert!(canonicalize_input_files(&input_files, work_dir, true).is_err());
}

#[test]
fn test_expand_input_files_with_kcl_mod() {
let path = PathBuf::from("src/test_data/expand_file_pattern");
let input_files = vec![
path.join("**").join("main.k").to_string_lossy().to_string(),
"${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(),
];
let expected_files = vec![
path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(),
path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(),
path.join("kcl1/main.k").to_string_lossy().to_string(),
path.join("kcl3/main.k").to_string_lossy().to_string(),
path.join("main.k").to_string_lossy().to_string(),
"${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(),
];
let got_paths: Vec<String> = expand_input_files(&input_files)
.iter()
.map(|s| s.replace(['/', '\\'], ""))
.collect();
let expect_paths: Vec<String> = expected_files
.iter()
.map(|s| s.replace(['/', '\\'], ""))
.collect();
assert_eq!(got_paths, expect_paths);
}

#[test]
#[cfg(not(windows))]
fn test_expand_input_files() {
let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()];
let mut expected_files = vec![
Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl3/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl1/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k")
.to_string_lossy()
.to_string(),
];
expected_files.sort();
let mut input = expand_input_files(&input_files);
input.sort();
assert_eq!(input, expected_files);

let input_files = vec![
"./src/test_data/expand_file_pattern/kcl1/main.k".to_string(),
"./src/test_data/expand_file_pattern/**/main.k".to_string(),
];
let mut expected_files = vec![
Path::new("src/test_data/expand_file_pattern/kcl1/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl1/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/kcl3/main.k")
.to_string_lossy()
.to_string(),
Path::new("src/test_data/expand_file_pattern/main.k")
.to_string_lossy()
.to_string(),
];
expected_files.sort();
let mut input = expand_input_files(&input_files);
input.sort();
assert_eq!(input, expected_files);
}
use crate::{get_pkg_list, lookup_the_nearest_file_dir, toolchain};

#[test]
fn test_parse_key_value_pair() {
Expand Down
1 change: 1 addition & 0 deletions kclvm/parser/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ regex = "1.7.0"
anyhow = "1.0"
indexmap = "1.0"
parking_lot = "0.12.3"
glob = "0.3.1"

kclvm-lexer = {path = "../lexer"}
kclvm-ast = {path = "../ast"}
Expand Down
Loading

0 comments on commit d61eeb2

Please sign in to comment.