refactor: input file expand function impl (#1646)

* refactor: input file expand function impl Signed-off-by: peefy <[email protected]> * chore: bump go version to 1.23 and add go apth to github path Signed-off-by: peefy <[email protected]> --------- Signed-off-by: peefy <[email protected]>
kcl-lang · Sep 13, 2024 · a00a1c6 · a00a1c6
1 parent 7344032
commit a00a1c6
Show file tree

Hide file tree

Showing 34 changed files with 227 additions and 270 deletions.
diff --git a/.github/workflows/build-test-macos-arm64.yml b/.github/workflows/build-test-macos-arm64.yml
@@ -14,11 +14,6 @@ jobs:
         with:
           submodules: "true"
 
-      - name: Set up Go 1.21
-        uses: actions/setup-go@v2
-        with:
-          go-version: 1.21
-
       - run: clang --version
       - run: cargo --version
       - run: rustc --print sysroot

diff --git a/.github/workflows/macos_test.yaml b/.github/workflows/macos_test.yaml
@@ -21,11 +21,14 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.21
+          go-version: 1.23
 
       - name: Install KCL CLI
-        run: go install kcl-lang.io/cli/cmd/kcl@main
-
+        run: |
+          go install kcl-lang.io/cli/cmd/kcl@latest
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+          echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH
+
       - run: clang --version
       - run: cargo --version
       - run: rustc --print sysroot

diff --git a/.github/workflows/ubuntu_test.yaml b/.github/workflows/ubuntu_test.yaml
@@ -13,7 +13,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.21
+          go-version: 1.23
 
       # Prerequisite
 
@@ -52,7 +52,10 @@ jobs:
         run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make test-runtime
         shell: bash
       - name: Install KCL CLI
-        run: go install kcl-lang.io/cli/cmd/kcl@main
+        run: |
+          go install kcl-lang.io/cli/cmd/kcl@latest
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+          echo "${{ github.workspace }}/go/bin" >> $GITHUB_PATH
 
       - name: Unit test
         working-directory: ./kclvm

diff --git a/.github/workflows/windows_test.yaml b/.github/workflows/windows_test.yaml
@@ -16,10 +16,16 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.21
+          go-version: 1.23
 
       - name: Install KCL
-        run: go install kcl-lang.io/cli/cmd/kcl@main
+        shell: powershell
+        run: |
+          go install kcl-lang.io/cli/cmd/kcl@latest
+          $GoPath = go env GOPATH
+          $GoInstallBin = Join-Path $GoPath "bin"
+          $Env:PATH += ";$GoInstallBin"
+          $Env:PATH += ";${{ github.workspace }}\go\bin"
 
       - uses: ilammy/msvc-dev-cmd@v1
 

diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock
diff --git a/kclvm/api/src/service/service_impl.rs b/kclvm/api/src/service/service_impl.rs
@@ -5,12 +5,11 @@ use std::string::String;
 
 use crate::gpyrpc::*;
 
-use anyhow::anyhow;
 use kcl_language_server::rename;
 use kclvm_config::settings::build_settings_pathbuf;
-use kclvm_driver::canonicalize_input_files;
 use kclvm_loader::option::list_options;
 use kclvm_loader::{load_packages_with_cache, LoadPackageOptions};
+use kclvm_parser::entry::get_normalized_k_files_from_paths;
 use kclvm_parser::load_program;
 use kclvm_parser::parse_file;
 use kclvm_parser::KCLModuleCache;
@@ -861,12 +860,13 @@ impl KclvmServiceImpl {
         let settings_files = args.files.iter().map(|f| f.as_str()).collect::<Vec<&str>>();
         let settings_pathbuf = build_settings_pathbuf(&[], Some(settings_files), None)?;
         let files = if !settings_pathbuf.settings().input().is_empty() {
-            canonicalize_input_files(
+            get_normalized_k_files_from_paths(
                 &settings_pathbuf.settings().input(),
-                args.work_dir.clone(),
-                false,
-            )
-            .map_err(|e| anyhow!(e))?
+                &LoadProgramOptions {
+                    work_dir: args.work_dir.clone(),
+                    ..Default::default()
+                },
+            )?
         } else {
             vec![]
         };

diff --git a/kclvm/driver/Cargo.toml b/kclvm/driver/Cargo.toml
@@ -15,10 +15,9 @@ kclvm-utils ={ path = "../utils"}
 kclvm-parser ={ path = "../parser"}
 kclvm-ast ={ path = "../ast"}
 walkdir = "2"
-
 serde = { version = "1.0", features = ["derive"] }
 anyhow = { version = "1.0.70", features = ["backtrace"] }
-glob = "0.3.1"
+
 flate2 = "1.0.30"
 tar = "0.4.40"
 indexmap = "2.2.6"

diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs
@@ -7,13 +7,11 @@ pub mod toolchain;
 mod tests;
 
 use anyhow::Result;
-use glob::glob;
 use kclvm_config::{
     modfile::{
         get_pkg_root, load_mod_file, KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_FILE,
-        KCL_MOD_PATH_ENV, KCL_WORK_FILE,
+        KCL_WORK_FILE,
     },
-    path::ModRelativePath,
     settings::{build_settings_pathbuf, DEFAULT_SETTING_FILE},
     workfile::load_work_file,
 };
@@ -30,106 +28,6 @@ use std::{
 use toolchain::{fill_pkg_maps_for_k_file, Metadata, Toolchain};
 use walkdir::WalkDir;
 
-/// Expand the single file pattern to a list of files.
-pub fn expand_if_file_pattern(file_pattern: String) -> Result<Vec<String>, String> {
-    let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?;
-    let mut matched_files = vec![];
-
-    for path in paths.flatten() {
-        matched_files.push(path.to_string_lossy().to_string());
-    }
-
-    Ok(matched_files)
-}
-
-/// Expand input kcl files with the file patterns.
-pub fn expand_input_files(k_files: &[String]) -> Vec<String> {
-    let mut res = vec![];
-    for file in k_files {
-        if let Ok(files) = expand_if_file_pattern(file.to_string()) {
-            if !files.is_empty() {
-                res.extend(files);
-            } else {
-                res.push(file.to_string())
-            }
-        } else {
-            res.push(file.to_string())
-        }
-    }
-    res
-}
-
-/// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path.
-pub fn canonicalize_input_files(
-    k_files: &[String],
-    work_dir: String,
-    check_exist: bool,
-) -> Result<Vec<String>, String> {
-    let mut kcl_paths = Vec::<String>::new();
-    // The first traversal changes the relative path to an absolute path
-    for file in k_files.iter() {
-        let path = Path::new(file);
-
-        let is_absolute = path.is_absolute();
-        let is_exist_maybe_symlink = path.exists();
-        // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS,
-        // join with the work directory path and convert it to a absolute path.
-        let path = ModRelativePath::from(file.to_string());
-        let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? {
-            let filepath = Path::new(&work_dir).join(file);
-            match filepath.canonicalize() {
-                Ok(path) => Some(path.adjust_canonicalization()),
-                Err(_) => {
-                    if check_exist {
-                        return Err(format!(
-                            "Cannot find the kcl file, please check the file path {}",
-                            file
-                        ));
-                    }
-                    Some(filepath.to_string_lossy().to_string())
-                }
-            }
-        } else {
-            None
-        };
-        // If the input file or path is a symlink, convert it to a real path.
-        let real_path = if is_exist_maybe_symlink {
-            match PathBuf::from(file.to_string()).canonicalize() {
-                Ok(real_path) => Some(String::from(real_path.to_str().unwrap())),
-                Err(_) => {
-                    if check_exist {
-                        return Err(format!(
-                            "Cannot find the kcl file, please check the file path {}",
-                            file
-                        ));
-                    }
-                    Some(file.to_string())
-                }
-            }
-        } else {
-            None
-        };
-
-        kcl_paths.push(abs_path.unwrap_or(real_path.unwrap_or(file.to_string())));
-    }
-
-    // Get the root path of the project
-    let pkgroot = kclvm_config::modfile::get_pkg_root_from_paths(&kcl_paths, work_dir)?;
-
-    // The second traversal replaces ${KCL_MOD} with the project root path
-    kcl_paths = kcl_paths
-        .iter()
-        .map(|file| {
-            if file.contains(KCL_MOD_PATH_ENV) {
-                file.replace(KCL_MOD_PATH_ENV, pkgroot.as_str())
-            } else {
-                file.clone()
-            }
-        })
-        .collect();
-    Ok(kcl_paths)
-}
-
 /// Get compile workspace(files and options) from a single file input.
 /// 1. Lookup entry files in kcl.yaml
 /// 2. Lookup entry files in kcl.mod
@@ -183,15 +81,12 @@ pub fn lookup_compile_workspace(
                         work_dir: work_dir.clone(),
                         ..Default::default()
                     };
-                    match canonicalize_input_files(&files, work_dir, true) {
-                        Ok(kcl_paths) => {
-                            // 1. find the kcl.mod path
-                            let metadata =
-                                fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt)
-                                    .unwrap_or(None);
-                            (kcl_paths, Some(load_opt), metadata)
-                        }
-                        Err(_) => default_res,
+                    let metadata =
+                        fill_pkg_maps_for_k_file(tool, file.into(), &mut load_opt).unwrap_or(None);
+                    if files.is_empty() {
+                        default_res
+                    } else {
+                        (files, Some(load_opt), metadata)
                     }
                 }
                 Err(_) => default_res,
@@ -205,10 +100,7 @@ pub fn lookup_compile_workspace(
                 if let Some(files) = mod_file.get_entries() {
                     let work_dir = dir.to_string_lossy().to_string();
                     load_opt.work_dir = work_dir.clone();
-                    match canonicalize_input_files(&files, work_dir, true) {
-                        Ok(kcl_paths) => (kcl_paths, Some(load_opt), metadata),
-                        Err(_) => default_res,
-                    }
+                    (files, Some(load_opt), metadata)
                 } else {
                     default_res
                 }
@@ -267,7 +159,6 @@ pub fn lookup_compile_workspaces(
                     (vec![path.to_string()], Some(load_opt), metadata),
                 );
             }
-
             WorkSpaceKind::SettingFile(setting_file) => {
                 workspaces.insert(
                     workspace.clone(),
@@ -278,7 +169,6 @@ pub fn lookup_compile_workspaces(
                     ),
                 );
             }
-
             WorkSpaceKind::ModFile(mod_file) => {
                 workspaces.insert(
                     workspace.clone(),
@@ -289,7 +179,6 @@ pub fn lookup_compile_workspaces(
                     ),
                 );
             }
-
             WorkSpaceKind::File(_) | WorkSpaceKind::NotFound => {
                 let pathbuf = PathBuf::from(path);
                 let file_path = pathbuf.as_path();