Skip to content

Commit

Permalink
grammar test
Browse files Browse the repository at this point in the history 
Signed-off-by: he1pa <[email protected]>
  • Loading branch information
@He1pa
He1pa committed Sep 19, 2024
1 parent ff81f90 commit ad80400
Show file tree
Hide file tree
Showing 2 changed files with 153 additions and 56 deletions.
104 changes: 92 additions & 12 deletions kclvm/parser/src/file_graph.rs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,21 @@
use indexmap::IndexMap;
use petgraph::visit::EdgeRef;
use std::{collections::HashMap, path::PathBuf};

use crate::File;
use indexmap::IndexMap;
use petgraph::{prelude::StableDiGraph, visit::EdgeRef};
/// File with package info
#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct PkgFile {
pub path: PathBuf,
pub pkg_name: String,
pub pkg_path: String,
pub pkg_root: String,
}

/// A graph of files, where each file depends on zero or more other files.
#[derive(Default)]
pub struct FileGraph {
graph: petgraph::stable_graph::StableDiGraph<File, ()>,
path_to_node_index: IndexMap<File, petgraph::graph::NodeIndex>,
graph: StableDiGraph<PkgFile, ()>,
path_to_node_index: IndexMap<PkgFile, petgraph::graph::NodeIndex>,
}

impl FileGraph {
Expand All @@ -16,9 +24,9 @@ impl FileGraph {
/// For example, if the current graph has file A depending on B, and
/// `update_file(pathA, &[pathC])` was called, then this function will remove the edge
/// from A to B, and add an edge from A to C.
pub fn update_file<'a, I: IntoIterator<Item = &'a File>>(
pub fn update_file<'a, I: IntoIterator<Item = &'a PkgFile>>(
&mut self,
from_path: &File,
from_path: &PkgFile,
to_paths: I,
) {
let from_node_index = self.get_or_insert_node_index(from_path);
Expand All @@ -40,14 +48,14 @@ impl FileGraph {
}

/// Returns true if the given file is in the graph
pub fn contains_file(&self, file: &File) -> bool {
pub fn contains_file(&self, file: &PkgFile) -> bool {
self.path_to_node_index.contains_key(file)
}

/// Returns a list of the direct dependencies of the given file.
/// (does not include all transitive dependencies)
/// The file path must be relative to the root of the file graph.
pub fn dependencies_of(&self, file: &File) -> Vec<&File> {
pub fn dependencies_of(&self, file: &PkgFile) -> Vec<&PkgFile> {
let node_index = self
.path_to_node_index
.get(file)
Expand All @@ -60,7 +68,7 @@ impl FileGraph {

/// Returns a list of files in the order they should be compiled
/// Or a list of files that are part of a cycle, if one exists
pub fn toposort(&self) -> Result<Vec<File>, Vec<File>> {
pub fn toposort(&self) -> Result<Vec<PkgFile>, Vec<PkgFile>> {
match petgraph::algo::toposort(&self.graph, None) {
Ok(indices) => Ok(indices
.into_iter()
Expand Down Expand Up @@ -91,11 +99,11 @@ impl FileGraph {

/// Returns all paths.
#[inline]
pub fn paths(&self) -> Vec<File> {
pub fn paths(&self) -> Vec<PkgFile> {
self.path_to_node_index.keys().cloned().collect::<Vec<_>>()
}

fn get_or_insert_node_index(&mut self, file: &File) -> petgraph::graph::NodeIndex {
fn get_or_insert_node_index(&mut self, file: &PkgFile) -> petgraph::graph::NodeIndex {
if let Some(node_index) = self.path_to_node_index.get(file) {
return *node_index;
}
Expand All @@ -104,4 +112,76 @@ impl FileGraph {
self.path_to_node_index.insert(file.to_owned(), node_index);
node_index
}

pub fn file_path_graph(&self) -> StableDiGraph<PathBuf, ()> {
let mut graph = StableDiGraph::new();
let mut node_map = HashMap::new();
for node in self.graph.node_indices() {
let path = self.graph[node].path.clone();
let idx = graph.add_node(path.clone());
node_map.insert(path, idx);
}
for edge in self.graph.edge_indices() {
if let Some((source, target)) = self.graph.edge_endpoints(edge) {
let source_path = self.graph[source].path.clone();
let target_path = self.graph[target].path.clone();
graph.add_edge(
node_map.get(&source_path).unwrap().clone(),
node_map.get(&target_path).unwrap().clone(),
(),
);
}
}
graph
}

pub fn pkg_graph(&self) -> StableDiGraph<String, ()> {
let mut graph = StableDiGraph::new();
let mut node_map = HashMap::new();
for node in self.graph.node_indices() {
let path = self.graph[node].pkg_path.clone();
let idx = graph.add_node(path.clone());
node_map.insert(path, idx);
}
for edge in self.graph.edge_indices() {
if let Some((source, target)) = self.graph.edge_endpoints(edge) {
let source_path = self.graph[source].pkg_path.clone();
let target_path = self.graph[target].pkg_path.clone();
graph.add_edge(
node_map.get(&source_path).unwrap().clone(),
node_map.get(&target_path).unwrap().clone(),
(),
);
}
}
graph
}
}

pub fn toposort<T>(graph: &StableDiGraph<T, ()>) -> Result<Vec<T>, Vec<T>>
where
T: Clone,
{
match petgraph::algo::toposort(graph, None) {
Ok(indices) => Ok(indices
.into_iter()
.rev()
.map(|n| graph[n].clone())
.collect::<Vec<_>>()),
Err(err) => {
// toposort function in the `petgraph` library doesn't return the cycle itself,
// so we need to use Tarjan's algorithm to find one instead
let strongly_connected_components = petgraph::algo::tarjan_scc(&graph);
// a strongly connected component is a cycle if it has more than one node
// let's just return the first one we find
let cycle = match strongly_connected_components
.into_iter()
.find(|component| component.len() > 1)
{
Some(vars) => vars,
None => vec![err.node_id()],
};
Err(cycle.iter().map(|n| graph[*n].clone()).collect::<Vec<_>>())
}
}
}
105 changes: 61 additions & 44 deletions kclvm/parser/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ pub use crate::session::{ParseSession, ParseSessionRef};
use compiler_base_macros::bug;
use compiler_base_session::Session;
use compiler_base_span::span::new_byte_pos;
use file_graph::FileGraph;
use file_graph::{toposort, FileGraph, PkgFile};
use indexmap::IndexMap;
use kclvm_ast::ast::Module;
use kclvm_ast::{ast, MAIN_PKG};
Expand Down Expand Up @@ -94,7 +94,7 @@ pub struct ParseFileResult {
/// Parse errors
pub errors: Errors,
/// Dependency paths.
pub deps: Vec<File>,
pub deps: Vec<PkgFile>,
}

/// Parse a KCL file to the AST module with parse errors.
Expand Down Expand Up @@ -544,7 +544,7 @@ impl Loader {
pkgs: &mut HashMap<String, Vec<ast::Module>>,
) -> Result<()> {
for m in pkg {
let mut to_paths: Vec<File> = vec![];
let mut to_paths: Vec<PkgFile> = vec![];
for stmt in &mut m.body {
let pos = stmt.pos().clone();
if let ast::Stmt::Import(ref mut import_spec) = &mut stmt.node {
Expand All @@ -567,10 +567,10 @@ impl Loader {
import_spec.path.node = pkg_info.pkg_path.to_string();
import_spec.pkg_name = pkg_info.pkg_name.clone();
// Add file dependencies.
let mut paths: Vec<File> = pkg_info
let mut paths: Vec<PkgFile> = pkg_info
.k_files
.iter()
.map(|p| File {
.map(|p| PkgFile {
path: p.into(),
pkg_name: pkg_info.pkg_name.clone(),
pkg_path: pkg_info.pkg_path.clone(),
Expand All @@ -582,7 +582,7 @@ impl Loader {
}
}
self.file_graph.write().unwrap().update_file(
&File {
&PkgFile {
path: m.filename.clone().into(),
pkg_name: pkg_name.clone(),
pkg_root: pkgroot.into(),
Expand Down Expand Up @@ -797,6 +797,28 @@ fn find_packages(
return Ok(None);
}

// plugin pkgs
if is_plugin_pkg(pkg_path) {
if !opts.load_plugins {
sess.1.write().add_error(
ErrorKind::CannotFindModule,
&[Message {
range: Into::<Range>::into(pos),
style: Style::Line,
message: format!("the plugin package `{}` is not found, please confirm if plugin mode is enabled", pkg_path),
note: None,
suggested_replacement: None,
}],
);
}
return Ok(None);
}

// builtin pkgs
if is_builtin_pkg(pkg_path) {
return Ok(None);
}

// 1. Look for in the current package's directory.
let is_internal = is_internal_pkg(pkg_name, pkg_root, pkg_path)?;
// 2. Look for in the vendor path.
Expand All @@ -821,6 +843,7 @@ fn find_packages(
}

// 4. Get package information based on whether the package is internal or external.

match is_internal.or(is_external) {
Some(pkg_info) => Ok(Some(pkg_info)),
None => {
Expand Down Expand Up @@ -1009,24 +1032,16 @@ fn is_external_pkg(pkg_path: &str, opts: LoadProgramOptions) -> Result<Option<Pk
pub type ASTCache = Arc<RwLock<IndexMap<PathBuf, Arc<ast::Module>>>>;
pub type FileGraphCache = Arc<RwLock<FileGraph>>;

#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct File {
pub path: PathBuf,
pub pkg_name: String,
pub pkg_path: String,
pub pkg_root: String,
}

pub fn parse_kcl_file(
sess: ParseSessionRef,
file: File,
file: PkgFile,
src: Option<String>,
asts: ASTCache,
file_graph: FileGraphCache,
opts: &LoadProgramOptions,
) -> Result<Vec<File>> {
) -> Result<Vec<PkgFile>> {
let mut m = parse_file_with_session(sess.clone(), file.path.to_str().unwrap(), src)?;
let mut deps: Vec<File> = vec![];
let mut deps: Vec<PkgFile> = vec![];
for stmt in &mut m.body {
let pos = stmt.pos().clone();
if let ast::Stmt::Import(ref mut import_spec) = &mut stmt.node {
Expand All @@ -1050,10 +1065,10 @@ pub fn parse_kcl_file(
import_spec.path.node = pkg_info.pkg_path.to_string();
import_spec.pkg_name = pkg_info.pkg_name.clone();
// Add file dependencies.
let mut paths: Vec<File> = pkg_info
let mut paths: Vec<PkgFile> = pkg_info
.k_files
.iter()
.map(|p| File {
.map(|p| PkgFile {
path: p.into(),
pkg_name: pkg_info.pkg_name.clone(),
pkg_root: pkg_info.pkg_root.clone().into(),
Expand All @@ -1072,11 +1087,11 @@ pub fn parse_kcl_file(

pub fn parse_kcl_pkg(
sess: ParseSessionRef,
files: Vec<(File, Option<String>)>,
files: Vec<(PkgFile, Option<String>)>,
asts: ASTCache,
file_graph: FileGraphCache,
opts: &LoadProgramOptions,
) -> Result<Vec<File>> {
) -> Result<Vec<PkgFile>> {
let mut dependent = vec![];
for (file, src) in files {
let deps = parse_kcl_file(
Expand Down Expand Up @@ -1104,7 +1119,7 @@ pub fn parse_kcl_entry(
let mut files = vec![];
for (i, f) in k_files.iter().enumerate() {
files.push((
File {
PkgFile {
path: f.into(),
pkg_name: entry.name().clone(),
pkg_root: entry.path().into(),
Expand Down Expand Up @@ -1153,31 +1168,33 @@ pub fn parse_kcl_program(
let file_graph = file_graph.read().unwrap();
let files = match file_graph.toposort() {
Ok(files) => files,
Err(cycle) => {
let formatted_cycle = cycle
.iter()
.map(|file| format!("- {}\n", file.path.to_string_lossy()))
.collect::<String>();
Err(_) => file_graph.paths(),
};

sess.1.write().add_error(
ErrorKind::RecursiveLoad,
&[Message {
range: (Position::dummy_pos(), Position::dummy_pos()),
style: Style::Line,
message: format!(
"Could not compiles due to cyclic import statements\n{}",
formatted_cycle.trim_end()
),
note: None,
suggested_replacement: None,
}],
);
let file_path_graph = file_graph.file_path_graph();
if let Err(cycle) = toposort(&file_path_graph) {
let formatted_cycle = cycle
.iter()
.map(|file| format!("- {}\n", file.to_string_lossy()))
.collect::<String>();

sess.1.write().add_error(
ErrorKind::RecursiveLoad,
&[Message {
range: (Position::dummy_pos(), Position::dummy_pos()),
style: Style::Line,
message: format!(
"Could not compiles due to cyclic import statements\n{}",
formatted_cycle.trim_end()
),
note: None,
suggested_replacement: None,
}],
);
}

// Return a list of all paths.
file_graph.paths()
}
};
let mut pkgs: HashMap<String, Vec<Module>> = HashMap::new();

let asts = asts.read().unwrap();
for file in files.iter() {
let m = asts.get(&file.path).unwrap().as_ref().clone();
Expand Down

0 comments on commit ad80400

Please sign in to comment.