fix: 485 allow invalid extensions (#562)

crates/cli/src/analyze.rs

@@ -18,7 +18,7 @@ use indicatif::{MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle};
 use marzano_core::built_in_functions::BuiltIns;
 use marzano_core::pattern_compiler::{src_to_problem_libs, CompilationResult};
 use marzano_core::{
-    api::{AnalysisLog, DoneFile, MatchResult},
+    api::{AnalysisLog, MatchResult},
     problem::Problem,
 };
 use marzano_language::target_language::PatternLanguage;
@@ -205,49 +205,19 @@ where
                 if file.file_type().unwrap().is_dir() {
                     continue;
                 }
-                if !&compiled.language.match_extension(
-                    file.path()
+                if !my_input.paths.contains(&file.path().to_path_buf()) {
+                    let ext = file
+                        .path()
                         .extension()
                         .unwrap_or_default()
                         .to_str()
-                        .unwrap_or_default(),
-                ) {
-                    processed.fetch_add(1, Ordering::SeqCst);
-                    let path_string = file.path().to_string_lossy().to_string();
-                    if my_input.paths.contains(&file.path().to_path_buf()) {
-                        let log = MatchResult::AnalysisLog(AnalysisLog {
-                            level: 410,
-                            message: format!(
-                                "Skipped {} since it is not a {} file",
-                                path_string,
-                                &compiled.language.to_string()
-                            ),
-                            position: Position::first(),
-                            file: path_string.to_string(),
-                            engine_id: "marzano".to_string(),
-                            range: None,
-                            syntax_tree: None,
-                            source: None,
-                        });
-                        let done_file = MatchResult::DoneFile(DoneFile {
-                            relative_file_path: path_string,
-                            has_results: Some(false),
-                            file_hash: None,
-                            from_cache: false,
-                        });
-                        emitter.handle_results(
-                            vec![log, done_file],
-                            details,
-                            arg.dry_run,
-                            arg.format,
-                            &mut interactive,
-                            None,
-                            Some(processed),
-                            None,
-                            &compiled.language,
-                        );
+                        .unwrap_or_default();
+                    if !&compiled.language.match_extension(ext) {
+                        // only skip the file if it was discovered by the walker
+                        // don't skip if it was explicitly passed in as a path
+                        // https://github.com/getgrit/gritql/issues/485
+                        continue;
                     }
-                    continue;
                 }
                 file_paths_tx.send(file.path().to_path_buf()).unwrap();
             }

crates/cli_bin/fixtures/invalid_extensions/some_folder/file1.py

@@ -0,0 +1 @@
+class MyClass(object): ...

crates/cli_bin/fixtures/invalid_extensions/some_folder/file2.pyi

@@ -0,0 +1 @@
+class MyClass(object): ...

crates/cli_bin/fixtures/invalid_extensions/some_folder/file3.nopy

@@ -0,0 +1 @@
+class MyClass(object): ...

crates/cli_bin/fixtures/invalid_extensions/some_folder/file4.js.py

@@ -0,0 +1,3 @@
+const mine = (object) => {
+    return String(object) + 'is mine';
+};

crates/cli_bin/tests/apply.rs

@@ -2913,6 +2913,144 @@ fn apply_stdin_with_invalid_lang_alias() -> Result<()> {
     Ok(())
 }
 
+/// test that we can apply to a folder which contains valid and invalid python extensions
+/// see https://github.com/getgrit/gritql/issues/485
+#[test]
+fn apply_to_folder_with_invalid_python_extension() -> Result<()> {
+    let (_temp_dir, fixture_dir) = get_fixture("invalid_extensions", false)?;
+
+    let mut cmd = get_test_cmd()?;
+    cmd.arg("apply")
+        .arg("`object` => ``")
+        .arg("some_folder")
+        .arg("--lang=py")
+        .arg("--force")
+        .current_dir(&fixture_dir);
+
+    let result = cmd.output()?;
+
+    let stderr = String::from_utf8(result.stderr)?;
+    println!("stderr: {:?}", stderr);
+    let stdout = String::from_utf8(result.stdout)?;
+    println!("stdout: {:?}", stdout);
+
+    assert!(result.status.success(), "Command failed");
+    // Read back the file3.nopy file to ensure it was processed
+    let target_file = fixture_dir.join("some_folder/file3.nopy");
+    let content: String = fs_err::read_to_string(target_file)?;
+    assert_snapshot!(content);
+
+    // ensure we don't get the old error message:
+    assert!(!stdout.contains("file3.nopy: ERROR (code: 410)"));
+    assert!(stdout.contains("some_folder/file4.js.py: ERROR (code: 300) - Error parsing source code at 1:7 in some_folder/file4.js.py. This may cause otherwise applicable queries to not match"));
+    assert!(stdout.contains("Processed 3 files and found 4 matches"));
+
+    Ok(())
+}
+
+/// test that we can apply to a path with an invalid python extension
+/// see https://github.com/getgrit/gritql/issues/485
+#[test]
+fn apply_to_path_with_invalid_python_extension() -> Result<()> {
+    let (_temp_dir, fixture_dir) = get_fixture("invalid_extensions", false)?;
+
+    let mut cmd = get_test_cmd()?;
+    cmd.arg("apply")
+        .arg("`object` => ``")
+        .arg("some_folder/file1.py")
+        .arg("some_folder/file2.pyi")
+        .arg("some_folder/file3.nopy")
+        .arg("--lang=py")
+        .arg("--force")
+        .current_dir(&fixture_dir);
+
+    let result = cmd.output()?;
+
+    let stderr = String::from_utf8(result.stderr)?;
+    println!("stderr: {:?}", stderr);
+    let stdout = String::from_utf8(result.stdout)?;
+    println!("stdout: {:?}", stdout);
+
+    assert!(result.status.success(), "Command failed");
+    // Read back the file3.nopy file to ensure it was processed
+    let target_file = fixture_dir.join("some_folder/file3.nopy");
+    let content: String = fs_err::read_to_string(target_file)?;
+    assert_snapshot!(content);
+
+    // ensure we don't get the old error message:
+    assert!(!stdout.contains("file3.nopy: ERROR (code: 410)"));
+    assert!(stdout.contains("Processed 3 files and found 3 matches"));
+
+    Ok(())
+}
+
+/// test that we can apply to a path with an invalid javascript extension
+/// see https://github.com/getgrit/gritql/issues/485
+#[test]
+fn apply_to_path_with_invalid_javascript_extension() -> Result<()> {
+    let (_temp_dir, fixture_dir) = get_fixture("invalid_extensions", false)?;
+
+    let mut cmd = get_test_cmd()?;
+    cmd.arg("apply")
+        .arg("`object` => ``")
+        .arg("some_folder/file4.js.py")
+        .arg("--lang=js")
+        .arg("--force")
+        .current_dir(&fixture_dir);
+
+    let result = cmd.output()?;
+
+    let stderr = String::from_utf8(result.stderr)?;
+    println!("stderr: {:?}", stderr);
+    let stdout = String::from_utf8(result.stdout)?;
+    println!("stdout: {:?}", stdout);
+
+    assert!(result.status.success(), "Command failed");
+    // Read back the file4.js.py file to ensure it was processed
+    let target_file = fixture_dir.join("some_folder/file4.js.py");
+    let content: String = fs_err::read_to_string(target_file)?;
+    assert_snapshot!(content);
+
+    assert!(stdout.contains("Processed 1 files and found 2 matches"));
+
+    Ok(())
+}
+
+/// test that we show an 'Error parsing source code' when we try to apply
+/// to a path which contains the wrong language as specified in the lang flag
+/// see https://github.com/getgrit/gritql/issues/485
+#[test]
+fn apply_to_path_with_invalid_lang() -> Result<()> {
+    let (_temp_dir, fixture_dir) = get_fixture("invalid_extensions", false)?;
+
+    let mut cmd = get_test_cmd()?;
+    cmd.arg("apply")
+        .arg("`object` => ``")
+        .arg("some_folder/file4.js.py")
+        .arg("--lang=py")
+        .arg("--force")
+        .current_dir(&fixture_dir);
+
+    let result = cmd.output()?;
+
+    let stderr = String::from_utf8(result.stderr)?;
+    println!("stderr: {:?}", stderr);
+    let stdout = String::from_utf8(result.stdout)?;
+    println!("stdout: {:?}", stdout);
+
+    assert!(result.status.success(), "Command failed");
+    // Read back the file4.js.py file to ensure it was processed
+    let target_file = fixture_dir.join("some_folder/file4.js.py");
+    let content: String = fs_err::read_to_string(target_file)?;
+    assert_snapshot!(content);
+
+    // we should get an error message about the wrong language / Error parsing source code
+    assert!(stdout.contains("some_folder/file4.js.py: ERROR (code: 300) - Error parsing source code at 1:7 in some_folder/file4.js.py. This may cause otherwise applicable queries to not match."));
+    assert!(stdout.contains("Processed 1 files and found 2 matches"));
+
+    Ok(())
+}
+
 /// Ban multiple stdin paths
 #[test]
 fn apply_stdin_two_paths() -> Result<()> {

crates/cli_bin/tests/snapshots/apply__apply_to_folder_with_invalid_python_extension.snap

@@ -0,0 +1,5 @@
+---
+source: crates/cli_bin/tests/apply.rs
+expression: content
+---
+class MyClass(object): ...

crates/cli_bin/tests/snapshots/apply__apply_to_path_with_invalid_javascript_extension.snap

@@ -0,0 +1,7 @@
+---
+source: crates/cli_bin/tests/apply.rs
+expression: content
+---
+const mine = () => {
+    return String() + 'is mine';
+};

crates/cli_bin/tests/snapshots/apply__apply_to_path_with_invalid_lang.snap

@@ -0,0 +1,7 @@
+---
+source: crates/cli_bin/tests/apply.rs
+expression: content
+---
+const mine = () => {
+    return String() + 'is mine';
+};

crates/cli_bin/tests/snapshots/apply__apply_to_path_with_invalid_python_extension.snap

@@ -0,0 +1,5 @@
+---
+source: crates/cli_bin/tests/apply.rs
+expression: content
+---
+class MyClass(): ...

crates/language/src/target_language.rs

@@ -287,7 +287,7 @@ impl PatternLanguage {
             PatternLanguage::Json => &["json"],
             PatternLanguage::Java => &["java"],
             PatternLanguage::CSharp => &["cs"],
-            PatternLanguage::Python => &["py", "ipynb"],
+            PatternLanguage::Python => &["py", "pyi", "ipynb"],
             PatternLanguage::MarkdownBlock => &["md", "mdx", "mdoc"],
             PatternLanguage::MarkdownInline => &["md", "mdx", "mdoc"],
             PatternLanguage::Go => &["go"],
@@ -343,7 +343,7 @@ impl PatternLanguage {
             "java" => Some(Self::Java),
             "cs" => Some(Self::CSharp),
             "ipynb" => Some(Self::Python),
-            "py" => Some(Self::Python),
+            "py" | "pyi" => Some(Self::Python),
             "md" | "mdx" | "mdoc" => Some(Self::MarkdownBlock),
             "go" => Some(Self::Go),
             "rs" => Some(Self::Rust),