Merge pull request #111 from vcfxb/rework-parser

Rework parser - largely restart the project.

.github/actions-rs/grcov.yml

@@ -2,5 +2,7 @@ branch: true
 output-type: lcov
 output-file: ./lcov.info
 ignore-not-existing: true
+llvm: true
 ignore:
   - "/*"
+  - "../*"

.github/workflows/codecov-io.yml

@@ -0,0 +1,47 @@
+on:
+  push:
+    branches:
+  pull_request:
+    branches: 
+      - "main"
+env:
+  CARGO_TERM_COLOR: always
+
+name: codecov.io Code Coverage
+jobs:
+  coverage:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install LLVM
+        # See: https://apt.llvm.org/
+        # Last line: https://gitlab.com/taricorp/llvm-sys.rs/-/issues/13
+        run: |
+          wget https://apt.llvm.org/llvm.sh
+          chmod +x llvm.sh
+          sudo ./llvm.sh 17
+          sudo apt install libpolly-17-dev libz-dev
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+
+      - name: Run tests
+        run: cargo test --verbose
+        env:
+          CARGO_INCREMENTAL: '0'
+          RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+          RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
+
+      - name: rust-grcov
+        # You may pin to the exact commit or the version.
+        # uses: actions-rs/grcov@bb47b1ed7883a1502fa6875d562727ace2511248
+        uses: actions-rs/[email protected]
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/[email protected]
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          slug: vcfxb/wright-lang

.github/workflows/grcov.yml

@@ -1,6 +1,6 @@
 on: ["push", "pull_request"]
 
-name: Code Coverage
+name: coveralls Code Coverage
 
 jobs:
   coverage:
@@ -19,18 +19,17 @@ jobs:
         with:
           toolchain: nightly
           override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all-features --no-fail-fast
+      - name: Run tests
+        run: cargo test --verbose
         env:
           CARGO_INCREMENTAL: '0'
           RUSTFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
           RUSTDOCFLAGS: '-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=abort -Zpanic_abort_tests'
-      - id: coverage
+      - name: rust-grcov
+        id: coverage
         uses: actions-rs/[email protected]
       - name: Coveralls upload
         uses: coverallsapp/github-action@master
         with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ${{ steps.coverage.outputs.report }}

README.md

@@ -6,7 +6,8 @@
 | Cargo Check Status | ![Cargo Check status](https://github.com/vcfxb/wright-lang/actions/workflows/cargo-check.yml/badge.svg?branch=master) |
 | Cargo Test Status | ![Cargo Test status](https://github.com/vcfxb/wright-lang/actions/workflows/cargo-test.yml/badge.svg?branch=master) |
 | Cargo Clippy Status | ![Cargo Clippy status](https://github.com/vcfxb/wright-lang/actions/workflows/cargo-clippy.yml/badge.svg?branch=master) |
-| Code Coverage | [![Coverage Status](https://coveralls.io/repos/github/vcfxb/wright-lang/badge.svg?branch=master&kill_cache=1)](https://coveralls.io/github/vcfxb/wright-lang?branch=master) |
+| Code Coverage (Coveralls) | [![Coverage Status](https://coveralls.io/repos/github/vcfxb/wright-lang/badge.svg?branch=master&kill_cache=1)](https://coveralls.io/github/vcfxb/wright-lang?branch=master) |
+| Code Coverage (Codecov.io) | [![codecov](https://codecov.io/github/vcfxb/wright-lang/graph/badge.svg?token=HO07JEYMIH)](https://codecov.io/github/vcfxb/wright-lang) |
 | Docs.rs | [![Documentation](https://docs.rs/wright/badge.svg)](https://docs.rs/wright) |
 | Crates.io | [![Crates.io](https://img.shields.io/crates/v/wright.svg)](https://crates.io/crates/wright) |
 | GitHub release | [![GitHub release](https://img.shields.io/github/release/vcfxb/wright-lang.svg)](https://github.com/vcfxb/wright-lang/releases) |

wright/Cargo.toml

@@ -26,10 +26,11 @@ test = false
 doc = false
 doctest = false
 
-# BENCHMARKS (currently none)
-# [[bench]]
-# name = "lexer"
-# harness = false
+# BENCHMARKS
+[[bench]]
+name = "lexer"
+harness = false
+
 
 # CRATES.IO BADGES
 [badges.maintenance]
@@ -62,9 +63,9 @@ version = "0.11.1"
 [dependencies.termcolor]
 version = "1.2.0"
 
-# Big Integers
-[dependencies.num] 
-version = "0.4"
+# # Big Integers
+# [dependencies.num] 
+# version = "0.4"
 
 # Portable (windows, mac, linux) file locking
 [dependencies.fs4] 
@@ -86,16 +87,12 @@ features = ["strict-versioning", "force-static"]
 version = "0.3"
 features = ["llvm17-0"]
 
-# # Fast parsing for integers and floats from source code. 
-# [dependencies.lexical-core]
-# version = "0.8"
-
 # TEST DEPENDENCIES
 
-# Rayon to speed up brute-force testing in some cases.
-[dev-dependencies.rayon]
-version = "1.8.0"
-
 # Criterion is used for benchmarking. 
 [dev-dependencies.criterion] 
 version = "0.5.1"
+
+# Rayon is used to do various brute-force tests in parallel
+[dev-dependencies.rayon]
+version = "1.8.0"

wright/benches/lexer.rs

@@ -0,0 +1,31 @@
+//! Lexer benchmarks.
+
+use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion};
+use wright::parser::lexer::Lexer;
+
+fn bench_symbol_tokens(c: &mut Criterion) {
+    // Make a benchmark group.
+    let mut group = c.benchmark_group("lexer symbol benchmarks");
+
+    // Function to make a lexer and get a token from it.
+    fn make_lexer_and_get_token(b: &mut Bencher, input: &str) {
+        b.iter(|| black_box(Lexer::new(input).next_token()));
+    }
+
+    let inputs = ["+", "+=", "*", "@", "?"];
+
+    for i in inputs {
+        group.bench_with_input(format!("lexer {i}"), i, make_lexer_and_get_token);
+    }
+}
+
+fn bench_block_doc_comment(c: &mut Criterion) {
+    c.bench_function("lexer block style doc comment", move |b: &mut Bencher| {
+        b.iter(move || {
+            black_box(Lexer::new("/*! \n this is a block-style comment \n\n */").next_token())
+        });
+    });
+}
+
+criterion_group!(benches, bench_symbol_tokens, bench_block_doc_comment);
+criterion_main!(benches);

wright/src/bin/wright.rs

@@ -2,17 +2,25 @@
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use codespan_reporting::files::SimpleFile;
-use std::{fs, path::PathBuf};
-use wright::{parser::lexer::Lexer, repl};
+use codespan_reporting::files::Files;
+use std::{path::PathBuf, time::Instant};
+use wright::{
+    filemap::{FileId, FileMap},
+    parser::lexer::{token::Token, Lexer},
+    repl,
+};
 
 /// The wright cli.
 #[derive(Parser, Debug)]
-#[command(author, version, about, long_about = None)]
+#[command(author, version, about, long_about = None, arg_required_else_help = true)]
 struct Cli {
     /// The subcommand passed to the wright cli.
     #[command(subcommand)]
     command: Option<Commands>,
+
+    /// Output elapsed timing information at the end of the command.
+    #[arg(short, long)]
+    timed: bool,
 }
 
 /// Different sub-commands that the wright cli supports.
@@ -35,39 +43,46 @@ enum DebugCommands {
     Tokens {
         /// A file of wright source code.
         file: PathBuf,
-
-        /// Pretty print the source code with the tokens lined under them.
-        /// If not used, a list of tokens will be printed with their metadata.
-        #[arg(short, long)]
-        pretty: bool,
+        // /// Pretty print the source code with the tokens lined under them.
+        // /// If not used, a list of tokens will be printed with their metadata.
+        // #[arg(short, long)]
+        // pretty: bool,
     },
 }
 
 fn main() -> Result<()> {
-    let cli = Cli::parse();
+    // Parse the command line arguments.
+    let cli: Cli = Cli::parse();
+    // Get the start time to track duration if asked.
+    let start: Instant = Instant::now();
 
     match cli.command {
-        // Printing token debug information.
+        // Start an interactive repl.
+        Some(Commands::Repl) => {
+            repl::start()?;
+        }
+
+        // Print all the tokens for a given file.
         Some(Commands::Debug {
-            command: DebugCommands::Tokens { file, pretty },
+            command: DebugCommands::Tokens { file },
         }) => {
-            let source_str = fs::read_to_string(&file)?;
-            let source = SimpleFile::new(file.to_string_lossy(), &source_str);
-
-            if pretty {
-                Lexer::debug_pretty_print(&source)?;
-            } else {
-                for token in Lexer::new(&source_str) {
-                    println!("{}", token);
-                }
-            }
-
-            Ok(())
+            let mut file_map: FileMap = FileMap::new();
+            // Add the given file to the file map.
+            let file_id: FileId = file_map.add_file(file)?;
+            // Make a lexer over the entirety of the given file.
+            // Use unwrap here, since we know we just added the file.
+            let lexer: Lexer = Lexer::new(file_map.source(file_id).unwrap());
+            // Get all the tokens from the lexer and print them each.
+            lexer.for_each(|token: Token| println!("{token}"));
         }
 
-        // Start an interactive repl.
-        Some(Commands::Repl) => repl::start(),
-
         _ => unimplemented!(),
     }
+
+    // Handle timing info.
+    if cli.timed {
+        println!("\nTotal time elapsed since parsing arguments: {:?}", Instant::now() - start);
+    }
+
+    Ok(())
 }