diff --git a/.github/workflows/build-test-macos-arm64.yml b/.github/workflows/build-test-macos-arm64.yml index 1234f4aa4..15ec91a30 100644 --- a/.github/workflows/build-test-macos-arm64.yml +++ b/.github/workflows/build-test-macos-arm64.yml @@ -14,6 +14,11 @@ jobs: with: submodules: "true" + - name: Set up Go 1.21 + uses: actions/setup-go@v2 + with: + go-version: 1.21 + - run: clang --version - run: cargo --version - run: rustc --print sysroot diff --git a/.github/workflows/macos_test.yaml b/.github/workflows/macos_test.yaml index 6d7096d46..ec4c1d8d5 100644 --- a/.github/workflows/macos_test.yaml +++ b/.github/workflows/macos_test.yaml @@ -21,14 +21,11 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.23 + go-version: 1.21 - name: Install KCL CLI - run: | - go install kcl-lang.io/cli/cmd/kcl@main - echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH - + run: go install kcl-lang.io/cli/cmd/kcl@main + - run: clang --version - run: cargo --version - run: rustc --print sysroot diff --git a/.github/workflows/ubuntu_test.yaml b/.github/workflows/ubuntu_test.yaml index 25558a2bd..cf907faa9 100644 --- a/.github/workflows/ubuntu_test.yaml +++ b/.github/workflows/ubuntu_test.yaml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.23 + go-version: 1.21 # Prerequisite @@ -52,10 +52,7 @@ jobs: run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make test-runtime shell: bash - name: Install KCL CLI - run: | - go install kcl-lang.io/cli/cmd/kcl@main - echo "$(go env GOPATH)/bin" >> $GITHUB_PATH - echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH + run: go install kcl-lang.io/cli/cmd/kcl@main - name: Unit test working-directory: ./kclvm diff --git a/.github/workflows/windows_test.yaml b/.github/workflows/windows_test.yaml index 9e25cb4b9..40aa986e1 100644 --- a/.github/workflows/windows_test.yaml +++ b/.github/workflows/windows_test.yaml @@ -16,16 +16,10 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.23 + go-version: 1.21 - name: Install KCL - shell: powershell - run: | - go install kcl-lang.io/cli/cmd/kcl@main - $GoPath = go env GOPATH - $GoInstallBin = Join-Path $GoPath "bin" - $Env:PATH += ";$GoInstallBin" - $Env:PATH += ";${{ github.workspace }}\go\bin" + run: go install kcl-lang.io/cli/cmd/kcl@latest - uses: ilammy/msvc-dev-cmd@v1 diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 6460e4b7c..af0f12015 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -1907,6 +1907,7 @@ version = "0.10.0-rc.1" dependencies = [ "anyhow", "flate2", + "glob", "indexmap 2.2.6", "kclvm-ast", "kclvm-config", @@ -2011,7 +2012,6 @@ dependencies = [ "either", "enquote", "expect-test", - "glob", "indexmap 1.9.3", "insta", "kclvm-ast", diff --git a/kclvm/api/src/service/service_impl.rs b/kclvm/api/src/service/service_impl.rs index e06aedf70..387173d32 100644 --- a/kclvm/api/src/service/service_impl.rs +++ b/kclvm/api/src/service/service_impl.rs @@ -5,11 +5,12 @@ use std::string::String; use crate::gpyrpc::*; +use anyhow::anyhow; use kcl_language_server::rename; use kclvm_config::settings::build_settings_pathbuf; +use kclvm_driver::canonicalize_input_files; use kclvm_loader::option::list_options; use kclvm_loader::{load_packages_with_cache, LoadPackageOptions}; -use kclvm_parser::entry::get_normalized_k_files_from_paths; use kclvm_parser::load_program; use kclvm_parser::parse_file; use kclvm_parser::KCLModuleCache; @@ -860,13 +861,12 @@ impl KclvmServiceImpl { let settings_files = args.files.iter().map(|f| f.as_str()).collect::>(); let settings_pathbuf = build_settings_pathbuf(&[], Some(settings_files), None)?; let files = if !settings_pathbuf.settings().input().is_empty() { - get_normalized_k_files_from_paths( + canonicalize_input_files( &settings_pathbuf.settings().input(), - &LoadProgramOptions { - work_dir: args.work_dir.clone(), - ..Default::default() - }, - )? + args.work_dir.clone(), + false, + ) + .map_err(|e| anyhow!(e))? } else { vec![] }; diff --git a/kclvm/driver/Cargo.toml b/kclvm/driver/Cargo.toml index 79a57d9eb..8fa9cf3cb 100644 --- a/kclvm/driver/Cargo.toml +++ b/kclvm/driver/Cargo.toml @@ -15,9 +15,10 @@ kclvm-utils ={ path = "../utils"} kclvm-parser ={ path = "../parser"} kclvm-ast ={ path = "../ast"} walkdir = "2" + serde = { version = "1.0", features = ["derive"] } anyhow = { version = "1.0.70", features = ["backtrace"] } - +glob = "0.3.1" flate2 = "1.0.30" tar = "0.4.40" indexmap = "2.2.6" diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index fb6326dd7..91d43374d 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -7,11 +7,13 @@ pub mod toolchain; mod tests; use anyhow::Result; +use glob::glob; use kclvm_config::{ modfile::{ get_pkg_root, load_mod_file, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE, - KCL_WORK_FILE, + KCL_MOD_PATH_ENV, KCL_WORK_FILE, }, + path::ModRelativePath, settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE}, workfile::load_work_file, }; @@ -28,6 +30,106 @@ use std::{ use toolchain::{fill_pkg_maps_for_k_file, Metadata, Toolchain}; use walkdir::WalkDir; +/// Expand the single file pattern to a list of files. +pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { + let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?; + let mut matched_files = vec![]; + + for path in paths.flatten() { + matched_files.push(path.to_string_lossy().to_string()); + } + + Ok(matched_files) +} + +/// Expand input kcl files with the file patterns. +pub fn expand_input_files(k_files: &[String]) -> Vec { + let mut res = vec![]; + for file in k_files { + if let Ok(files) = expand_if_file_pattern(file.to_string()) { + if !files.is_empty() { + res.extend(files); + } else { + res.push(file.to_string()) + } + } else { + res.push(file.to_string()) + } + } + res +} + +/// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path. +pub fn canonicalize_input_files( + k_files: &[String], + work_dir: String, + check_exist: bool, +) -> Result, String> { + let mut kcl_paths = Vec::::new(); + // The first traversal changes the relative path to an absolute path + for file in k_files.iter() { + let path = Path::new(file); + + let is_absolute = path.is_absolute(); + let is_exist_maybe_symlink = path.exists(); + // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, + // join with the work directory path and convert it to a absolute path. + let path = ModRelativePath::from(file.to_string()); + let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? { + let filepath = Path::new(&work_dir).join(file); + match filepath.canonicalize() { + Ok(path) => Some(path.adjust_canonicalization()), + Err(_) => { + if check_exist { + return Err(format!( + "Cannot find the kcl file, please check the file path {}", + file + )); + } + Some(filepath.to_string_lossy().to_string()) + } + } + } else { + None + }; + // If the input file or path is a symlink, convert it to a real path. + let real_path = if is_exist_maybe_symlink { + match PathBuf::from(file.to_string()).canonicalize() { + Ok(real_path) => Some(String::from(real_path.to_str().unwrap())), + Err(_) => { + if check_exist { + return Err(format!( + "Cannot find the kcl file, please check the file path {}", + file + )); + } + Some(file.to_string()) + } + } + } else { + None + }; + + kcl_paths.push(abs_path.unwrap_or(real_path.unwrap_or(file.to_string()))); + } + + // Get the root path of the project + let pkgroot = kclvm_config::modfile::get_pkg_root_from_paths(&kcl_paths, work_dir)?; + + // The second traversal replaces ${KCL_MOD} with the project root path + kcl_paths = kcl_paths + .iter() + .map(|file| { + if file.contains(KCL_MOD_PATH_ENV) { + file.replace(KCL_MOD_PATH_ENV, pkgroot.as_str()) + } else { + file.clone() + } + }) + .collect(); + Ok(kcl_paths) +} + /// Get compile workspace(files and options) from a single file input. /// 1. Lookup entry files in kcl.yaml /// 2. Lookup entry files in kcl.mod @@ -81,12 +183,15 @@ pub fn lookup_compile_workspace( work_dir: work_dir.clone(), ..Default::default() }; - let metadata = - fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt).unwrap_or(None); - if files.is_empty() { - default_res - } else { - (files, Some(load_opt), metadata) + match canonicalize_input_files(&files, work_dir, true) { + Ok(kcl_paths) => { + // 1. find the kcl.mod path + let metadata = + fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt) + .unwrap_or(None); + (kcl_paths, Some(load_opt), metadata) + } + Err(_) => default_res, } } Err(_) => default_res, @@ -100,7 +205,10 @@ pub fn lookup_compile_workspace( if let Some(files) = mod_file.get_entries() { let work_dir = dir.to_string_lossy().to_string(); load_opt.work_dir = work_dir.clone(); - (files, Some(load_opt), metadata) + match canonicalize_input_files(&files, work_dir, true) { + Ok(kcl_paths) => (kcl_paths, Some(load_opt), metadata), + Err(_) => default_res, + } } else { default_res } @@ -159,6 +267,7 @@ pub fn lookup_compile_workspaces( (vec![path.to_string()], Some(load_opt), metadata), ); } + WorkSpaceKind::SettingFile(setting_file) => { workspaces.insert( workspace.clone(), @@ -169,6 +278,7 @@ pub fn lookup_compile_workspaces( ), ); } + WorkSpaceKind::ModFile(mod_file) => { workspaces.insert( workspace.clone(), @@ -179,6 +289,7 @@ pub fn lookup_compile_workspaces( ), ); } + WorkSpaceKind::File(_) | WorkSpaceKind::NotFound => { let pathbuf = PathBuf::from(path); let file_path = pathbuf.as_path(); diff --git a/kclvm/parser/testdata/expand_file_pattern/KCL_MOD b/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/KCL_MOD rename to kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl.mod rename to kclvm/driver/src/test_data/expand_file_pattern/kcl.mod diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/kcl.mod rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/kcl.mod rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/main.k rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/kcl.mod rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/main.k rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl1/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl1/main.k rename to kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl3/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl3/kcl.mod rename to kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod diff --git a/kclvm/parser/testdata/expand_file_pattern/kcl3/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/kcl3/main.k rename to kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k diff --git a/kclvm/parser/testdata/expand_file_pattern/main.k b/kclvm/driver/src/test_data/expand_file_pattern/main.k similarity index 100% rename from kclvm/parser/testdata/expand_file_pattern/main.k rename to kclvm/driver/src/test_data/expand_file_pattern/main.k diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index fb68bf670..07844ad8b 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::{env, fs, panic}; use kclvm_config::modfile::get_vendor_home; @@ -9,7 +9,105 @@ use walkdir::WalkDir; use crate::arguments::parse_key_value_pair; use crate::toolchain::Toolchain; use crate::toolchain::{fill_pkg_maps_for_k_file, CommandToolchain, NativeToolchain}; -use crate::{get_pkg_list, lookup_the_nearest_file_dir, toolchain}; +use crate::{canonicalize_input_files, expand_input_files, get_pkg_list}; +use crate::{lookup_the_nearest_file_dir, toolchain}; + +#[test] +fn test_canonicalize_input_files() { + let input_files = vec!["file1.k".to_string(), "file2.k".to_string()]; + let work_dir = ".".to_string(); + let expected_files = vec![ + Path::new(".").join("file1.k").to_string_lossy().to_string(), + Path::new(".").join("file2.k").to_string_lossy().to_string(), + ]; + assert_eq!( + canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), + expected_files + ); + assert!(canonicalize_input_files(&input_files, work_dir, true).is_err()); +} + +#[test] +fn test_expand_input_files_with_kcl_mod() { + let path = PathBuf::from("src/test_data/expand_file_pattern"); + let input_files = vec![ + path.join("**").join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), + ]; + let expected_files = vec![ + path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), + path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), + path.join("kcl1/main.k").to_string_lossy().to_string(), + path.join("kcl3/main.k").to_string_lossy().to_string(), + path.join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), + ]; + let got_paths: Vec = expand_input_files(&input_files) + .iter() + .map(|s| s.replace(['/', '\\'], "")) + .collect(); + let expect_paths: Vec = expected_files + .iter() + .map(|s| s.replace(['/', '\\'], "")) + .collect(); + assert_eq!(got_paths, expect_paths); +} + +#[test] +#[cfg(not(windows))] +fn test_expand_input_files() { + let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; + let mut expected_files = vec![ + Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl3/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .to_string_lossy() + .to_string(), + ]; + expected_files.sort(); + let mut input = expand_input_files(&input_files); + input.sort(); + assert_eq!(input, expected_files); + + let input_files = vec![ + "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), + "./src/test_data/expand_file_pattern/**/main.k".to_string(), + ]; + let mut expected_files = vec![ + Path::new("src/test_data/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/kcl3/main.k") + .to_string_lossy() + .to_string(), + Path::new("src/test_data/expand_file_pattern/main.k") + .to_string_lossy() + .to_string(), + ]; + expected_files.sort(); + let mut input = expand_input_files(&input_files); + input.sort(); + assert_eq!(input, expected_files); +} #[test] fn test_parse_key_value_pair() { diff --git a/kclvm/parser/Cargo.toml b/kclvm/parser/Cargo.toml index c3320ea80..c5fef777a 100644 --- a/kclvm/parser/Cargo.toml +++ b/kclvm/parser/Cargo.toml @@ -24,7 +24,6 @@ regex = "1.7.0" anyhow = "1.0" indexmap = "1.0" parking_lot = "0.12.3" -glob = "0.3.1" kclvm-lexer = {path = "../lexer"} kclvm-ast = {path = "../ast"} diff --git a/kclvm/parser/src/entry.rs b/kclvm/parser/src/entry.rs index 2b53f85dc..6cec6f9f3 100644 --- a/kclvm/parser/src/entry.rs +++ b/kclvm/parser/src/entry.rs @@ -1,5 +1,4 @@ use anyhow::Result; -use glob::glob; use kclvm_config::modfile::get_pkg_root; use kclvm_config::modfile::KCL_FILE_SUFFIX; use kclvm_config::path::ModRelativePath; @@ -298,10 +297,8 @@ pub fn get_compile_entries_from_paths( } let mut result = Entries::default(); let mut k_code_queue = VecDeque::from(opts.k_code_list.clone()); - let file_paths = expand_input_files(file_paths); - for file in &file_paths { - let file = canonicalize_input_file(file, &opts.work_dir); - let path = ModRelativePath::from(file.to_string()); + for s in file_paths { + let path = ModRelativePath::from(s.to_string()); // If the path is a [`ModRelativePath`] with prefix '${:KCL_MOD}', // calculate the real path and the package name. @@ -311,11 +308,11 @@ pub fn get_compile_entries_from_paths( .map(|pkg_path: &String| (name, pkg_path)) }) { // Replace the mod relative path prefix '${:KCL_MOD}' with the real path. - let file = path.canonicalize_by_root_path(pkg_path)?; - if let Some(root) = get_pkg_root(&file) { + let s = path.canonicalize_by_root_path(pkg_path)?; + if let Some(root) = get_pkg_root(&s) { let mut entry: Entry = Entry::new(pkg_name.clone(), root.clone()); entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(&file, &root, &pkg_name, opts)?, + get_main_files_from_pkg_path(&s, &root, &pkg_name, opts)?, &mut k_code_queue, ); result.push_entry(entry); @@ -328,11 +325,11 @@ pub fn get_compile_entries_from_paths( entry.push_k_code(k_code_queue.pop_front()); result.push_entry(entry); continue; - } else if let Some(root) = get_pkg_root(&file) { + } else if let Some(root) = get_pkg_root(s) { // If the path is a normal path. let mut entry: Entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), root.clone()); entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(&file, &root, kclvm_ast::MAIN_PKG, opts)?, + get_main_files_from_pkg_path(s, &root, kclvm_ast::MAIN_PKG, opts)?, &mut k_code_queue, ); result.push_entry(entry); @@ -345,9 +342,9 @@ pub fn get_compile_entries_from_paths( .is_empty() { let mut entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), "".to_string()); - for file in &file_paths { + for s in file_paths { entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(&file, "", kclvm_ast::MAIN_PKG, opts)?, + get_main_files_from_pkg_path(s, "", kclvm_ast::MAIN_PKG, opts)?, &mut k_code_queue, ); } @@ -518,56 +515,3 @@ fn is_ignored_file(filename: &str) -> bool { || (filename.ends_with("_test.k")) || (filename.starts_with('_')) } - -/// Normalize the input file with the working directory and replace ${KCL_MOD} with the module root path. -fn canonicalize_input_file(file: &str, work_dir: &str) -> String { - let path = std::path::Path::new(file); - let is_absolute = path.is_absolute(); - // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, - // join with the work directory path and convert it to a absolute path. - let path = ModRelativePath::from(file.to_string()); - let is_relative_path = match path.is_relative_path() { - Ok(is_relative_path) => is_relative_path, - _ => return file.to_string(), - }; - - let abs_path = if !is_absolute && !is_relative_path { - let filepath = std::path::Path::new(work_dir).join(file); - match filepath.canonicalize() { - Ok(path) => Some(path.adjust_canonicalization()), - Err(_) => Some(filepath.to_string_lossy().to_string()), - } - } else { - None - }; - abs_path.unwrap_or(file.to_string()) -} - -/// Expand the single file pattern to a list of files. -pub fn expand_if_file_pattern(file_pattern: String) -> Result> { - let paths = glob(&file_pattern)?; - let mut matched_files = vec![]; - - for path in paths.flatten() { - matched_files.push(path.to_string_lossy().to_string()); - } - - Ok(matched_files) -} - -/// Expand input kcl files with the file patterns e.g. `/path/to/kcl/*.k`, `**/**/*.k` -pub fn expand_input_files(k_files: &[String]) -> Vec { - let mut res = vec![]; - for file in k_files { - if let Ok(files) = expand_if_file_pattern(file.to_string()) { - if !files.is_empty() { - res.extend(files); - } else { - res.push(file.to_string()) - } - } else { - res.push(file.to_string()) - } - } - res -} diff --git a/kclvm/parser/src/tests.rs b/kclvm/parser/src/tests.rs index 68e4e41e6..f378d233d 100644 --- a/kclvm/parser/src/tests.rs +++ b/kclvm/parser/src/tests.rs @@ -4,7 +4,6 @@ use std::{ }; use compiler_base_span::{FilePathMapping, SourceMap}; -use entry::expand_input_files; use kclvm_config::modfile::{get_vendor_home, KCL_PKG_PATH}; use crate::*; @@ -738,85 +737,3 @@ pub fn test_pkg_not_found_suggestion() { } } } - -#[test] -fn test_expand_input_files_with_kcl_mod() { - let path = PathBuf::from("testdata/expand_file_pattern"); - let input_files = vec![ - path.join("**").join("main.k").to_string_lossy().to_string(), - "${KCL_MOD}/testdata/expand_file_pattern/KCL_MOD".to_string(), - ]; - let expected_files = vec![ - path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), - path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), - path.join("kcl1/main.k").to_string_lossy().to_string(), - path.join("kcl3/main.k").to_string_lossy().to_string(), - path.join("main.k").to_string_lossy().to_string(), - "${KCL_MOD}/testdata/expand_file_pattern/KCL_MOD".to_string(), - ]; - let got_paths: Vec = expand_input_files(&input_files) - .iter() - .map(|s| s.replace(['/', '\\'], "")) - .collect(); - let expect_paths: Vec = expected_files - .iter() - .map(|s| s.replace(['/', '\\'], "")) - .collect(); - assert_eq!(got_paths, expect_paths); -} - -#[test] -#[cfg(not(windows))] -fn test_expand_input_files() { - let input_files = vec!["./testdata/expand_file_pattern/**/main.k".to_string()]; - let mut expected_files = vec![ - Path::new("testdata/expand_file_pattern/kcl1/kcl2/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl3/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl1/kcl4/main.k") - .to_string_lossy() - .to_string(), - ]; - expected_files.sort(); - let mut input = expand_input_files(&input_files); - input.sort(); - assert_eq!(input, expected_files); - - let input_files = vec![ - "./testdata/expand_file_pattern/kcl1/main.k".to_string(), - "./testdata/expand_file_pattern/**/main.k".to_string(), - ]; - let mut expected_files = vec![ - Path::new("testdata/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl1/kcl2/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl1/kcl4/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/kcl3/main.k") - .to_string_lossy() - .to_string(), - Path::new("testdata/expand_file_pattern/main.k") - .to_string_lossy() - .to_string(), - ]; - expected_files.sort(); - let mut input = expand_input_files(&input_files); - input.sort(); - assert_eq!(input, expected_files); -} diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index 4d361976c..dde68dcc3 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -7,6 +7,7 @@ use kclvm_ast::{ MAIN_PKG, }; use kclvm_config::cache::KCL_CACHE_PATH_ENV_VAR; +use kclvm_driver::{canonicalize_input_files, expand_input_files}; use kclvm_parser::{load_program, KCLModuleCache, ParseSessionRef}; use kclvm_query::apply_overrides; use kclvm_sema::resolver::{ @@ -77,11 +78,8 @@ pub const KCL_FAST_EVAL_ENV_VAR: &str = "KCL_FAST_EVAL"; pub fn exec_program(sess: ParseSessionRef, args: &ExecProgramArgs) -> Result { // parse args from json string let opts = args.get_load_program_options(); - let kcl_paths_str = args - .k_filename_list - .iter() - .map(|s| s.as_str()) - .collect::>(); + let kcl_paths = expand_files(args)?; + let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); let module_cache = KCLModuleCache::default(); let mut program = load_program( sess.clone(), @@ -272,11 +270,8 @@ pub fn build_program>( ) -> Result { // Parse program. let opts = args.get_load_program_options(); - let kcl_paths_str = args - .k_filename_list - .iter() - .map(|s| s.as_str()) - .collect::>(); + let kcl_paths = expand_files(args)?; + let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); let mut program = load_program(sess.clone(), kcl_paths_str.as_slice(), Some(opts), None)?.program; // Resolve program. @@ -352,6 +347,16 @@ fn build>( Artifact::from_path(lib_path) } +/// Expand and return the normalized file paths for the input file list. +pub fn expand_files(args: &ExecProgramArgs) -> Result> { + let k_files = &args.k_filename_list; + let work_dir = args.work_dir.clone().unwrap_or_default(); + let k_files = expand_input_files(k_files); + let kcl_paths = + canonicalize_input_files(&k_files, work_dir, false).map_err(|err| anyhow!(err))?; + Ok(kcl_paths) +} + /// Clean all the tmp files generated during lib generating and linking. #[inline] #[cfg(feature = "llvm")] diff --git a/kclvm/tools/src/LSP/src/state.rs b/kclvm/tools/src/LSP/src/state.rs index 25c5824cd..4ee297c90 100644 --- a/kclvm/tools/src/LSP/src/state.rs +++ b/kclvm/tools/src/LSP/src/state.rs @@ -264,10 +264,6 @@ impl LanguageServerState { // If all workspaces do not contain the current file, get files workspace and store in temporary_workspace if !may_contain { - self.log_message(format!( - "Not contains in any workspace, compile: {:?}", - filename - )); let tool = Arc::clone(&self.tool); let (workspaces, failed) = lookup_compile_workspaces(&*tool.read(), &filename, true); diff --git a/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod b/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod index 4ae72f07f..8ea06569b 100644 --- a/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod +++ b/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod @@ -2,3 +2,4 @@ [dependencies] k8s = { oci = "oci://ghcr.io/kcl-lang/k8s", tag = "1.28" } + diff --git a/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod index 7afba4cdb..05c442f22 100644 --- a/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod +++ b/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod @@ -5,3 +5,4 @@ version = "0.0.4" [dependencies] konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" } + diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/base/base.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/base/base.k deleted file mode 100644 index e69de29bb..000000000 diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod deleted file mode 100644 index ef51dad31..000000000 --- a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod +++ /dev/null @@ -1,2 +0,0 @@ -[package] -name = "pkg_mod_test" diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k deleted file mode 100644 index 26dee57bd..000000000 --- a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k +++ /dev/null @@ -1,3 +0,0 @@ -import pkg2 -schema Name: - name: str \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg2/b.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg2/b.k deleted file mode 100644 index e69de29bb..000000000 diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod deleted file mode 100644 index 37fddc5ec..000000000 --- a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "pkg_mod" - -[dependencies] -pkg_mod_test = { path = "../../pkg_mod_test" } - -[profile] -entries = ["../base/base.k", "main.k", "${pkg_mod_test:KCL_MOD}/pkg1/a.k"] \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k deleted file mode 100644 index d8bd4746a..000000000 --- a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k +++ /dev/null @@ -1,3 +0,0 @@ -import pkg_mod_test - -import pkg_mod_test.pkg2 \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/tests.rs b/kclvm/tools/src/LSP/src/tests.rs index 610f65386..6a91ca916 100644 --- a/kclvm/tools/src/LSP/src/tests.rs +++ b/kclvm/tools/src/LSP/src/tests.rs @@ -2226,7 +2226,7 @@ fn kcl_workspace_init_kclmod_test() { } #[test] -fn kcl_workspace_init_settings_file_test() { +fn kcl_workspace_init_kclsettings_test() { let tool: crate::state::KCLToolChain = Arc::new(RwLock::new(toolchain::default())); let tool = Arc::clone(&tool); @@ -2258,11 +2258,11 @@ fn kcl_workspace_init_settings_file_test() { )); assert_eq!(expected, workspaces.keys().cloned().collect()); - assert!(failed.is_none()); assert_eq!( - vec![a.file_name().unwrap().to_str().unwrap().to_string()], + vec![a.to_str().unwrap().to_string(),], workspaces.values().next().unwrap().0 ); + assert!(failed.is_none()); } #[test] @@ -2394,10 +2394,3 @@ fn init_workspace_sema_token_test() { let res = server.send_and_receive(r); assert!(res.result.is_some()); } - -#[test] -fn pkg_mod_test() { - let (_file, _program, diags, _gs) = - compile_test_file("src/test_data/workspace/pkg_mod_test/test/main.k"); - assert_eq!(diags.iter().filter(|diag| diag.is_error()).count(), 0); -}