From a00a1c6c3e56ab70e5dbff2c8f0331bc5e406351 Mon Sep 17 00:00:00 2001 From: Peefy Date: Fri, 13 Sep 2024 14:34:44 +0800 Subject: [PATCH] refactor: input file expand function impl (#1646) * refactor: input file expand function impl Signed-off-by: peefy * chore: bump go version to 1.23 and add go apth to github path Signed-off-by: peefy --------- Signed-off-by: peefy --- .github/workflows/build-test-macos-arm64.yml | 5 - .github/workflows/macos_test.yaml | 9 +- .github/workflows/ubuntu_test.yaml | 7 +- .github/workflows/windows_test.yaml | 10 +- kclvm/Cargo.lock | 2 +- kclvm/api/src/service/service_impl.rs | 14 +- kclvm/driver/Cargo.toml | 3 +- kclvm/driver/src/lib.rs | 127 ++---------------- kclvm/driver/src/tests.rs | 102 +------------- kclvm/parser/Cargo.toml | 1 + kclvm/parser/src/entry.rs | 74 ++++++++-- kclvm/parser/src/tests.rs | 83 ++++++++++++ .../testdata}/expand_file_pattern/KCL_MOD | 0 .../testdata}/expand_file_pattern/kcl.mod | 0 .../expand_file_pattern/kcl1/kcl.mod | 0 .../expand_file_pattern/kcl1/kcl2/kcl.mod | 0 .../expand_file_pattern/kcl1/kcl2/main.k | 0 .../expand_file_pattern/kcl1/kcl4/kcl.mod | 0 .../expand_file_pattern/kcl1/kcl4/main.k | 0 .../testdata}/expand_file_pattern/kcl1/main.k | 0 .../expand_file_pattern/kcl3/kcl.mod | 0 .../testdata}/expand_file_pattern/kcl3/main.k | 0 .../testdata}/expand_file_pattern/main.k | 0 kclvm/runner/src/lib.rs | 25 ++-- kclvm/tools/src/LSP/src/state.rs | 4 + .../import/external/external_0/kcl.mod | 1 - .../test_data/goto_import_def_test/kcl.mod | 1 - .../workspace/pkg_mod_test/base/base.k | 0 .../test_data/workspace/pkg_mod_test/kcl.mod | 2 + .../test_data/workspace/pkg_mod_test/pkg1/a.k | 3 + .../test_data/workspace/pkg_mod_test/pkg2/b.k | 0 .../workspace/pkg_mod_test/test/kcl.mod | 8 ++ .../workspace/pkg_mod_test/test/main.k | 3 + kclvm/tools/src/LSP/src/tests.rs | 13 +- 34 files changed, 227 insertions(+), 270 deletions(-) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/KCL_MOD (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl.mod (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/kcl.mod (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/kcl2/kcl.mod (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/kcl2/main.k (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/kcl4/kcl.mod (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/kcl4/main.k (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl1/main.k (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl3/kcl.mod (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/kcl3/main.k (100%) rename kclvm/{driver/src/test_data => parser/testdata}/expand_file_pattern/main.k (100%) create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/base/base.k create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg2/b.k create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod create mode 100644 kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k diff --git a/.github/workflows/build-test-macos-arm64.yml b/.github/workflows/build-test-macos-arm64.yml index 15ec91a30..1234f4aa4 100644 --- a/.github/workflows/build-test-macos-arm64.yml +++ b/.github/workflows/build-test-macos-arm64.yml @@ -14,11 +14,6 @@ jobs: with: submodules: "true" - - name: Set up Go 1.21 - uses: actions/setup-go@v2 - with: - go-version: 1.21 - - run: clang --version - run: cargo --version - run: rustc --print sysroot diff --git a/.github/workflows/macos_test.yaml b/.github/workflows/macos_test.yaml index ec4c1d8d5..645bc3bc4 100644 --- a/.github/workflows/macos_test.yaml +++ b/.github/workflows/macos_test.yaml @@ -21,11 +21,14 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.23 - name: Install KCL CLI - run: go install kcl-lang.io/cli/cmd/kcl@main - + run: | + go install kcl-lang.io/cli/cmd/kcl@latest + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH + - run: clang --version - run: cargo --version - run: rustc --print sysroot diff --git a/.github/workflows/ubuntu_test.yaml b/.github/workflows/ubuntu_test.yaml index cf907faa9..cdb1f356f 100644 --- a/.github/workflows/ubuntu_test.yaml +++ b/.github/workflows/ubuntu_test.yaml @@ -13,7 +13,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.23 # Prerequisite @@ -52,7 +52,10 @@ jobs: run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make test-runtime shell: bash - name: Install KCL CLI - run: go install kcl-lang.io/cli/cmd/kcl@main + run: | + go install kcl-lang.io/cli/cmd/kcl@latest + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH - name: Unit test working-directory: ./kclvm diff --git a/.github/workflows/windows_test.yaml b/.github/workflows/windows_test.yaml index ea187619f..c2304e89e 100644 --- a/.github/workflows/windows_test.yaml +++ b/.github/workflows/windows_test.yaml @@ -16,10 +16,16 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.21 + go-version: 1.23 - name: Install KCL - run: go install kcl-lang.io/cli/cmd/kcl@main + shell: powershell + run: | + go install kcl-lang.io/cli/cmd/kcl@latest + $GoPath = go env GOPATH + $GoInstallBin = Join-Path $GoPath "bin" + $Env:PATH += ";$GoInstallBin" + $Env:PATH += ";${{ github.workspace }}\go\bin" - uses: ilammy/msvc-dev-cmd@v1 diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index af0f12015..6460e4b7c 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -1907,7 +1907,6 @@ version = "0.10.0-rc.1" dependencies = [ "anyhow", "flate2", - "glob", "indexmap 2.2.6", "kclvm-ast", "kclvm-config", @@ -2012,6 +2011,7 @@ dependencies = [ "either", "enquote", "expect-test", + "glob", "indexmap 1.9.3", "insta", "kclvm-ast", diff --git a/kclvm/api/src/service/service_impl.rs b/kclvm/api/src/service/service_impl.rs index 387173d32..e06aedf70 100644 --- a/kclvm/api/src/service/service_impl.rs +++ b/kclvm/api/src/service/service_impl.rs @@ -5,12 +5,11 @@ use std::string::String; use crate::gpyrpc::*; -use anyhow::anyhow; use kcl_language_server::rename; use kclvm_config::settings::build_settings_pathbuf; -use kclvm_driver::canonicalize_input_files; use kclvm_loader::option::list_options; use kclvm_loader::{load_packages_with_cache, LoadPackageOptions}; +use kclvm_parser::entry::get_normalized_k_files_from_paths; use kclvm_parser::load_program; use kclvm_parser::parse_file; use kclvm_parser::KCLModuleCache; @@ -861,12 +860,13 @@ impl KclvmServiceImpl { let settings_files = args.files.iter().map(|f| f.as_str()).collect::>(); let settings_pathbuf = build_settings_pathbuf(&[], Some(settings_files), None)?; let files = if !settings_pathbuf.settings().input().is_empty() { - canonicalize_input_files( + get_normalized_k_files_from_paths( &settings_pathbuf.settings().input(), - args.work_dir.clone(), - false, - ) - .map_err(|e| anyhow!(e))? + &LoadProgramOptions { + work_dir: args.work_dir.clone(), + ..Default::default() + }, + )? } else { vec![] }; diff --git a/kclvm/driver/Cargo.toml b/kclvm/driver/Cargo.toml index 8fa9cf3cb..79a57d9eb 100644 --- a/kclvm/driver/Cargo.toml +++ b/kclvm/driver/Cargo.toml @@ -15,10 +15,9 @@ kclvm-utils ={ path = "../utils"} kclvm-parser ={ path = "../parser"} kclvm-ast ={ path = "../ast"} walkdir = "2" - serde = { version = "1.0", features = ["derive"] } anyhow = { version = "1.0.70", features = ["backtrace"] } -glob = "0.3.1" + flate2 = "1.0.30" tar = "0.4.40" indexmap = "2.2.6" diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 91d43374d..fb6326dd7 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -7,13 +7,11 @@ pub mod toolchain; mod tests; use anyhow::Result; -use glob::glob; use kclvm_config::{ modfile::{ get_pkg_root, load_mod_file, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE, - KCL_MOD_PATH_ENV, KCL_WORK_FILE, + KCL_WORK_FILE, }, - path::ModRelativePath, settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE}, workfile::load_work_file, }; @@ -30,106 +28,6 @@ use std::{ use toolchain::{fill_pkg_maps_for_k_file, Metadata, Toolchain}; use walkdir::WalkDir; -/// Expand the single file pattern to a list of files. -pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { - let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?; - let mut matched_files = vec![]; - - for path in paths.flatten() { - matched_files.push(path.to_string_lossy().to_string()); - } - - Ok(matched_files) -} - -/// Expand input kcl files with the file patterns. -pub fn expand_input_files(k_files: &[String]) -> Vec { - let mut res = vec![]; - for file in k_files { - if let Ok(files) = expand_if_file_pattern(file.to_string()) { - if !files.is_empty() { - res.extend(files); - } else { - res.push(file.to_string()) - } - } else { - res.push(file.to_string()) - } - } - res -} - -/// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path. -pub fn canonicalize_input_files( - k_files: &[String], - work_dir: String, - check_exist: bool, -) -> Result, String> { - let mut kcl_paths = Vec::::new(); - // The first traversal changes the relative path to an absolute path - for file in k_files.iter() { - let path = Path::new(file); - - let is_absolute = path.is_absolute(); - let is_exist_maybe_symlink = path.exists(); - // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, - // join with the work directory path and convert it to a absolute path. - let path = ModRelativePath::from(file.to_string()); - let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? { - let filepath = Path::new(&work_dir).join(file); - match filepath.canonicalize() { - Ok(path) => Some(path.adjust_canonicalization()), - Err(_) => { - if check_exist { - return Err(format!( - "Cannot find the kcl file, please check the file path {}", - file - )); - } - Some(filepath.to_string_lossy().to_string()) - } - } - } else { - None - }; - // If the input file or path is a symlink, convert it to a real path. - let real_path = if is_exist_maybe_symlink { - match PathBuf::from(file.to_string()).canonicalize() { - Ok(real_path) => Some(String::from(real_path.to_str().unwrap())), - Err(_) => { - if check_exist { - return Err(format!( - "Cannot find the kcl file, please check the file path {}", - file - )); - } - Some(file.to_string()) - } - } - } else { - None - }; - - kcl_paths.push(abs_path.unwrap_or(real_path.unwrap_or(file.to_string()))); - } - - // Get the root path of the project - let pkgroot = kclvm_config::modfile::get_pkg_root_from_paths(&kcl_paths, work_dir)?; - - // The second traversal replaces ${KCL_MOD} with the project root path - kcl_paths = kcl_paths - .iter() - .map(|file| { - if file.contains(KCL_MOD_PATH_ENV) { - file.replace(KCL_MOD_PATH_ENV, pkgroot.as_str()) - } else { - file.clone() - } - }) - .collect(); - Ok(kcl_paths) -} - /// Get compile workspace(files and options) from a single file input. /// 1. Lookup entry files in kcl.yaml /// 2. Lookup entry files in kcl.mod @@ -183,15 +81,12 @@ pub fn lookup_compile_workspace( work_dir: work_dir.clone(), ..Default::default() }; - match canonicalize_input_files(&files, work_dir, true) { - Ok(kcl_paths) => { - // 1. find the kcl.mod path - let metadata = - fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt) - .unwrap_or(None); - (kcl_paths, Some(load_opt), metadata) - } - Err(_) => default_res, + let metadata = + fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt).unwrap_or(None); + if files.is_empty() { + default_res + } else { + (files, Some(load_opt), metadata) } } Err(_) => default_res, @@ -205,10 +100,7 @@ pub fn lookup_compile_workspace( if let Some(files) = mod_file.get_entries() { let work_dir = dir.to_string_lossy().to_string(); load_opt.work_dir = work_dir.clone(); - match canonicalize_input_files(&files, work_dir, true) { - Ok(kcl_paths) => (kcl_paths, Some(load_opt), metadata), - Err(_) => default_res, - } + (files, Some(load_opt), metadata) } else { default_res } @@ -267,7 +159,6 @@ pub fn lookup_compile_workspaces( (vec![path.to_string()], Some(load_opt), metadata), ); } - WorkSpaceKind::SettingFile(setting_file) => { workspaces.insert( workspace.clone(), @@ -278,7 +169,6 @@ pub fn lookup_compile_workspaces( ), ); } - WorkSpaceKind::ModFile(mod_file) => { workspaces.insert( workspace.clone(), @@ -289,7 +179,6 @@ pub fn lookup_compile_workspaces( ), ); } - WorkSpaceKind::File(_) | WorkSpaceKind::NotFound => { let pathbuf = PathBuf::from(path); let file_path = pathbuf.as_path(); diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 07844ad8b..fb68bf670 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -1,4 +1,4 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::{env, fs, panic}; use kclvm_config::modfile::get_vendor_home; @@ -9,105 +9,7 @@ use walkdir::WalkDir; use crate::arguments::parse_key_value_pair; use crate::toolchain::Toolchain; use crate::toolchain::{fill_pkg_maps_for_k_file, CommandToolchain, NativeToolchain}; -use crate::{canonicalize_input_files, expand_input_files, get_pkg_list}; -use crate::{lookup_the_nearest_file_dir, toolchain}; - -#[test] -fn test_canonicalize_input_files() { - let input_files = vec!["file1.k".to_string(), "file2.k".to_string()]; - let work_dir = ".".to_string(); - let expected_files = vec![ - Path::new(".").join("file1.k").to_string_lossy().to_string(), - Path::new(".").join("file2.k").to_string_lossy().to_string(), - ]; - assert_eq!( - canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), - expected_files - ); - assert!(canonicalize_input_files(&input_files, work_dir, true).is_err()); -} - -#[test] -fn test_expand_input_files_with_kcl_mod() { - let path = PathBuf::from("src/test_data/expand_file_pattern"); - let input_files = vec![ - path.join("**").join("main.k").to_string_lossy().to_string(), - "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), - ]; - let expected_files = vec![ - path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), - path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), - path.join("kcl1/main.k").to_string_lossy().to_string(), - path.join("kcl3/main.k").to_string_lossy().to_string(), - path.join("main.k").to_string_lossy().to_string(), - "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), - ]; - let got_paths: Vec = expand_input_files(&input_files) - .iter() - .map(|s| s.replace(['/', '\\'], "")) - .collect(); - let expect_paths: Vec = expected_files - .iter() - .map(|s| s.replace(['/', '\\'], "")) - .collect(); - assert_eq!(got_paths, expect_paths); -} - -#[test] -#[cfg(not(windows))] -fn test_expand_input_files() { - let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; - let mut expected_files = vec![ - Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl3/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k") - .to_string_lossy() - .to_string(), - ]; - expected_files.sort(); - let mut input = expand_input_files(&input_files); - input.sort(); - assert_eq!(input, expected_files); - - let input_files = vec![ - "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), - "./src/test_data/expand_file_pattern/**/main.k".to_string(), - ]; - let mut expected_files = vec![ - Path::new("src/test_data/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl1/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl1/kcl2/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl1/kcl4/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/kcl3/main.k") - .to_string_lossy() - .to_string(), - Path::new("src/test_data/expand_file_pattern/main.k") - .to_string_lossy() - .to_string(), - ]; - expected_files.sort(); - let mut input = expand_input_files(&input_files); - input.sort(); - assert_eq!(input, expected_files); -} +use crate::{get_pkg_list, lookup_the_nearest_file_dir, toolchain}; #[test] fn test_parse_key_value_pair() { diff --git a/kclvm/parser/Cargo.toml b/kclvm/parser/Cargo.toml index c5fef777a..c3320ea80 100644 --- a/kclvm/parser/Cargo.toml +++ b/kclvm/parser/Cargo.toml @@ -24,6 +24,7 @@ regex = "1.7.0" anyhow = "1.0" indexmap = "1.0" parking_lot = "0.12.3" +glob = "0.3.1" kclvm-lexer = {path = "../lexer"} kclvm-ast = {path = "../ast"} diff --git a/kclvm/parser/src/entry.rs b/kclvm/parser/src/entry.rs index 6cec6f9f3..2b53f85dc 100644 --- a/kclvm/parser/src/entry.rs +++ b/kclvm/parser/src/entry.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use glob::glob; use kclvm_config::modfile::get_pkg_root; use kclvm_config::modfile::KCL_FILE_SUFFIX; use kclvm_config::path::ModRelativePath; @@ -297,8 +298,10 @@ pub fn get_compile_entries_from_paths( } let mut result = Entries::default(); let mut k_code_queue = VecDeque::from(opts.k_code_list.clone()); - for s in file_paths { - let path = ModRelativePath::from(s.to_string()); + let file_paths = expand_input_files(file_paths); + for file in &file_paths { + let file = canonicalize_input_file(file, &opts.work_dir); + let path = ModRelativePath::from(file.to_string()); // If the path is a [`ModRelativePath`] with prefix '${:KCL_MOD}', // calculate the real path and the package name. @@ -308,11 +311,11 @@ pub fn get_compile_entries_from_paths( .map(|pkg_path: &String| (name, pkg_path)) }) { // Replace the mod relative path prefix '${:KCL_MOD}' with the real path. - let s = path.canonicalize_by_root_path(pkg_path)?; - if let Some(root) = get_pkg_root(&s) { + let file = path.canonicalize_by_root_path(pkg_path)?; + if let Some(root) = get_pkg_root(&file) { let mut entry: Entry = Entry::new(pkg_name.clone(), root.clone()); entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(&s, &root, &pkg_name, opts)?, + get_main_files_from_pkg_path(&file, &root, &pkg_name, opts)?, &mut k_code_queue, ); result.push_entry(entry); @@ -325,11 +328,11 @@ pub fn get_compile_entries_from_paths( entry.push_k_code(k_code_queue.pop_front()); result.push_entry(entry); continue; - } else if let Some(root) = get_pkg_root(s) { + } else if let Some(root) = get_pkg_root(&file) { // If the path is a normal path. let mut entry: Entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), root.clone()); entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(s, &root, kclvm_ast::MAIN_PKG, opts)?, + get_main_files_from_pkg_path(&file, &root, kclvm_ast::MAIN_PKG, opts)?, &mut k_code_queue, ); result.push_entry(entry); @@ -342,9 +345,9 @@ pub fn get_compile_entries_from_paths( .is_empty() { let mut entry = Entry::new(kclvm_ast::MAIN_PKG.to_string(), "".to_string()); - for s in file_paths { + for file in &file_paths { entry.extend_k_files_and_codes( - get_main_files_from_pkg_path(s, "", kclvm_ast::MAIN_PKG, opts)?, + get_main_files_from_pkg_path(&file, "", kclvm_ast::MAIN_PKG, opts)?, &mut k_code_queue, ); } @@ -515,3 +518,56 @@ fn is_ignored_file(filename: &str) -> bool { || (filename.ends_with("_test.k")) || (filename.starts_with('_')) } + +/// Normalize the input file with the working directory and replace ${KCL_MOD} with the module root path. +fn canonicalize_input_file(file: &str, work_dir: &str) -> String { + let path = std::path::Path::new(file); + let is_absolute = path.is_absolute(); + // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, + // join with the work directory path and convert it to a absolute path. + let path = ModRelativePath::from(file.to_string()); + let is_relative_path = match path.is_relative_path() { + Ok(is_relative_path) => is_relative_path, + _ => return file.to_string(), + }; + + let abs_path = if !is_absolute && !is_relative_path { + let filepath = std::path::Path::new(work_dir).join(file); + match filepath.canonicalize() { + Ok(path) => Some(path.adjust_canonicalization()), + Err(_) => Some(filepath.to_string_lossy().to_string()), + } + } else { + None + }; + abs_path.unwrap_or(file.to_string()) +} + +/// Expand the single file pattern to a list of files. +pub fn expand_if_file_pattern(file_pattern: String) -> Result> { + let paths = glob(&file_pattern)?; + let mut matched_files = vec![]; + + for path in paths.flatten() { + matched_files.push(path.to_string_lossy().to_string()); + } + + Ok(matched_files) +} + +/// Expand input kcl files with the file patterns e.g. `/path/to/kcl/*.k`, `**/**/*.k` +pub fn expand_input_files(k_files: &[String]) -> Vec { + let mut res = vec![]; + for file in k_files { + if let Ok(files) = expand_if_file_pattern(file.to_string()) { + if !files.is_empty() { + res.extend(files); + } else { + res.push(file.to_string()) + } + } else { + res.push(file.to_string()) + } + } + res +} diff --git a/kclvm/parser/src/tests.rs b/kclvm/parser/src/tests.rs index f378d233d..68e4e41e6 100644 --- a/kclvm/parser/src/tests.rs +++ b/kclvm/parser/src/tests.rs @@ -4,6 +4,7 @@ use std::{ }; use compiler_base_span::{FilePathMapping, SourceMap}; +use entry::expand_input_files; use kclvm_config::modfile::{get_vendor_home, KCL_PKG_PATH}; use crate::*; @@ -737,3 +738,85 @@ pub fn test_pkg_not_found_suggestion() { } } } + +#[test] +fn test_expand_input_files_with_kcl_mod() { + let path = PathBuf::from("testdata/expand_file_pattern"); + let input_files = vec![ + path.join("**").join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/testdata/expand_file_pattern/KCL_MOD".to_string(), + ]; + let expected_files = vec![ + path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), + path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), + path.join("kcl1/main.k").to_string_lossy().to_string(), + path.join("kcl3/main.k").to_string_lossy().to_string(), + path.join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/testdata/expand_file_pattern/KCL_MOD".to_string(), + ]; + let got_paths: Vec = expand_input_files(&input_files) + .iter() + .map(|s| s.replace(['/', '\\'], "")) + .collect(); + let expect_paths: Vec = expected_files + .iter() + .map(|s| s.replace(['/', '\\'], "")) + .collect(); + assert_eq!(got_paths, expect_paths); +} + +#[test] +#[cfg(not(windows))] +fn test_expand_input_files() { + let input_files = vec!["./testdata/expand_file_pattern/**/main.k".to_string()]; + let mut expected_files = vec![ + Path::new("testdata/expand_file_pattern/kcl1/kcl2/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl3/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl1/kcl4/main.k") + .to_string_lossy() + .to_string(), + ]; + expected_files.sort(); + let mut input = expand_input_files(&input_files); + input.sort(); + assert_eq!(input, expected_files); + + let input_files = vec![ + "./testdata/expand_file_pattern/kcl1/main.k".to_string(), + "./testdata/expand_file_pattern/**/main.k".to_string(), + ]; + let mut expected_files = vec![ + Path::new("testdata/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl1/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl1/kcl2/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl1/kcl4/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/kcl3/main.k") + .to_string_lossy() + .to_string(), + Path::new("testdata/expand_file_pattern/main.k") + .to_string_lossy() + .to_string(), + ]; + expected_files.sort(); + let mut input = expand_input_files(&input_files); + input.sort(); + assert_eq!(input, expected_files); +} diff --git a/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD b/kclvm/parser/testdata/expand_file_pattern/KCL_MOD similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD rename to kclvm/parser/testdata/expand_file_pattern/KCL_MOD diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod b/kclvm/parser/testdata/expand_file_pattern/kcl.mod similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl.mod rename to kclvm/parser/testdata/expand_file_pattern/kcl.mod diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod b/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl.mod similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod rename to kclvm/parser/testdata/expand_file_pattern/kcl1/kcl.mod diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod b/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/kcl.mod similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod rename to kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/kcl.mod diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k b/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/main.k similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k rename to kclvm/parser/testdata/expand_file_pattern/kcl1/kcl2/main.k diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod b/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/kcl.mod similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod rename to kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/kcl.mod diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k b/kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/main.k similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k rename to kclvm/parser/testdata/expand_file_pattern/kcl1/kcl4/main.k diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k b/kclvm/parser/testdata/expand_file_pattern/kcl1/main.k similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k rename to kclvm/parser/testdata/expand_file_pattern/kcl1/main.k diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod b/kclvm/parser/testdata/expand_file_pattern/kcl3/kcl.mod similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod rename to kclvm/parser/testdata/expand_file_pattern/kcl3/kcl.mod diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k b/kclvm/parser/testdata/expand_file_pattern/kcl3/main.k similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k rename to kclvm/parser/testdata/expand_file_pattern/kcl3/main.k diff --git a/kclvm/driver/src/test_data/expand_file_pattern/main.k b/kclvm/parser/testdata/expand_file_pattern/main.k similarity index 100% rename from kclvm/driver/src/test_data/expand_file_pattern/main.k rename to kclvm/parser/testdata/expand_file_pattern/main.k diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index dde68dcc3..4d361976c 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -7,7 +7,6 @@ use kclvm_ast::{ MAIN_PKG, }; use kclvm_config::cache::KCL_CACHE_PATH_ENV_VAR; -use kclvm_driver::{canonicalize_input_files, expand_input_files}; use kclvm_parser::{load_program, KCLModuleCache, ParseSessionRef}; use kclvm_query::apply_overrides; use kclvm_sema::resolver::{ @@ -78,8 +77,11 @@ pub const KCL_FAST_EVAL_ENV_VAR: &str = "KCL_FAST_EVAL"; pub fn exec_program(sess: ParseSessionRef, args: &ExecProgramArgs) -> Result { // parse args from json string let opts = args.get_load_program_options(); - let kcl_paths = expand_files(args)?; - let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); + let kcl_paths_str = args + .k_filename_list + .iter() + .map(|s| s.as_str()) + .collect::>(); let module_cache = KCLModuleCache::default(); let mut program = load_program( sess.clone(), @@ -270,8 +272,11 @@ pub fn build_program>( ) -> Result { // Parse program. let opts = args.get_load_program_options(); - let kcl_paths = expand_files(args)?; - let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); + let kcl_paths_str = args + .k_filename_list + .iter() + .map(|s| s.as_str()) + .collect::>(); let mut program = load_program(sess.clone(), kcl_paths_str.as_slice(), Some(opts), None)?.program; // Resolve program. @@ -347,16 +352,6 @@ fn build>( Artifact::from_path(lib_path) } -/// Expand and return the normalized file paths for the input file list. -pub fn expand_files(args: &ExecProgramArgs) -> Result> { - let k_files = &args.k_filename_list; - let work_dir = args.work_dir.clone().unwrap_or_default(); - let k_files = expand_input_files(k_files); - let kcl_paths = - canonicalize_input_files(&k_files, work_dir, false).map_err(|err| anyhow!(err))?; - Ok(kcl_paths) -} - /// Clean all the tmp files generated during lib generating and linking. #[inline] #[cfg(feature = "llvm")] diff --git a/kclvm/tools/src/LSP/src/state.rs b/kclvm/tools/src/LSP/src/state.rs index 4ee297c90..25c5824cd 100644 --- a/kclvm/tools/src/LSP/src/state.rs +++ b/kclvm/tools/src/LSP/src/state.rs @@ -264,6 +264,10 @@ impl LanguageServerState { // If all workspaces do not contain the current file, get files workspace and store in temporary_workspace if !may_contain { + self.log_message(format!( + "Not contains in any workspace, compile: {:?}", + filename + )); let tool = Arc::clone(&self.tool); let (workspaces, failed) = lookup_compile_workspaces(&*tool.read(), &filename, true); diff --git a/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod b/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod index 8ea06569b..4ae72f07f 100644 --- a/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod +++ b/kclvm/tools/src/LSP/src/test_data/completion_test/import/external/external_0/kcl.mod @@ -2,4 +2,3 @@ [dependencies] k8s = { oci = "oci://ghcr.io/kcl-lang/k8s", tag = "1.28" } - diff --git a/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod index 05c442f22..7afba4cdb 100644 --- a/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod +++ b/kclvm/tools/src/LSP/src/test_data/goto_import_def_test/kcl.mod @@ -5,4 +5,3 @@ version = "0.0.4" [dependencies] konfig = { git = "https://github.com/awesome-kusion/konfig.git", tag = "v0.0.1" } - diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/base/base.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/base/base.k new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod new file mode 100644 index 000000000..ef51dad31 --- /dev/null +++ b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/kcl.mod @@ -0,0 +1,2 @@ +[package] +name = "pkg_mod_test" diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k new file mode 100644 index 000000000..26dee57bd --- /dev/null +++ b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg1/a.k @@ -0,0 +1,3 @@ +import pkg2 +schema Name: + name: str \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg2/b.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/pkg2/b.k new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod new file mode 100644 index 000000000..37fddc5ec --- /dev/null +++ b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/kcl.mod @@ -0,0 +1,8 @@ +[package] +name = "pkg_mod" + +[dependencies] +pkg_mod_test = { path = "../../pkg_mod_test" } + +[profile] +entries = ["../base/base.k", "main.k", "${pkg_mod_test:KCL_MOD}/pkg1/a.k"] \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k new file mode 100644 index 000000000..d8bd4746a --- /dev/null +++ b/kclvm/tools/src/LSP/src/test_data/workspace/pkg_mod_test/test/main.k @@ -0,0 +1,3 @@ +import pkg_mod_test + +import pkg_mod_test.pkg2 \ No newline at end of file diff --git a/kclvm/tools/src/LSP/src/tests.rs b/kclvm/tools/src/LSP/src/tests.rs index 6a91ca916..610f65386 100644 --- a/kclvm/tools/src/LSP/src/tests.rs +++ b/kclvm/tools/src/LSP/src/tests.rs @@ -2226,7 +2226,7 @@ fn kcl_workspace_init_kclmod_test() { } #[test] -fn kcl_workspace_init_kclsettings_test() { +fn kcl_workspace_init_settings_file_test() { let tool: crate::state::KCLToolChain = Arc::new(RwLock::new(toolchain::default())); let tool = Arc::clone(&tool); @@ -2258,11 +2258,11 @@ fn kcl_workspace_init_kclsettings_test() { )); assert_eq!(expected, workspaces.keys().cloned().collect()); + assert!(failed.is_none()); assert_eq!( - vec![a.to_str().unwrap().to_string(),], + vec![a.file_name().unwrap().to_str().unwrap().to_string()], workspaces.values().next().unwrap().0 ); - assert!(failed.is_none()); } #[test] @@ -2394,3 +2394,10 @@ fn init_workspace_sema_token_test() { let res = server.send_and_receive(r); assert!(res.result.is_some()); } + +#[test] +fn pkg_mod_test() { + let (_file, _program, diags, _gs) = + compile_test_file("src/test_data/workspace/pkg_mod_test/test/main.k"); + assert_eq!(diags.iter().filter(|diag| diag.is_error()).count(), 0); +}