diff --git a/.github/workflows/build-cppfront.yaml b/.github/workflows/build-cppfront.yaml deleted file mode 100644 index c831bc4807..0000000000 --- a/.github/workflows/build-cppfront.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: Multi-platform Build of cppfront -on: - pull_request: - branches-ignore: - - docs - push: - branches-ignore: - - docs - workflow_dispatch: - -jobs: - build-windows: - runs-on: windows-latest - steps: - - uses: actions/checkout@v3 - - uses: ilammy/msvc-dev-cmd@v1 - - name: Compiler name & version - run: cl.exe - - name: Build - run: cl.exe source/cppfront.cpp -std:c++latest -MD -EHsc -experimental:module -W4 -WX - build-unix-like: - strategy: - fail-fast: false - matrix: - runs-on: [ubuntu-22.04] - compiler: [g++-10, g++-11, g++-12, clang++-12, clang++-14] - cxx-std: ['c++20', 'c++2b'] - exclude: - # GCC 10 doesn't have support for c++23 - - compiler: g++-10 - cxx-std: 'c++2b' - # Clang 12 and 14 do not compile on 'c++2b' due to llvm/llvm-project#58206 - - compiler: clang++-12 - cxx-std: 'c++2b' - - compiler: clang++-14 - cxx-std: 'c++2b' - include: - - runs-on: macos-latest - compiler: clang++ - cxx-std: 'c++20' - - runs-on: ubuntu-22.04 - compiler: clang++-15 - cxx-std: 'c++20' - - runs-on: ubuntu-24.04 - compiler: clang++-16 - cxx-std: 'c++20' - - runs-on: ubuntu-24.04 - compiler: clang++-17 - cxx-std: 'c++20' - - runs-on: ubuntu-24.04 - compiler: clang++-18 - cxx-std: 'c++20' - - runs-on: ubuntu-24.04 - compiler: clang++-18 - cxx-std: 'c++23' - - runs-on: ubuntu-24.04 - compiler: g++-14 - cxx-std: 'c++2b' - runs-on: ${{ matrix.runs-on }} - env: - CXX: ${{ matrix.compiler }} - CXXFLAGS: -std=${{ matrix.cxx-std }} -Wall -Wextra -Wold-style-cast -Wunused-parameter -Wpedantic -Werror -pthread -Wno-unknown-warning -Wno-unknown-warning-option - steps: - - uses: actions/checkout@v3 - - name: Install compiler - if: startsWith(matrix.runs-on, 'ubuntu') - run: sudo apt-get install -y $CXX - - name: Compiler name & version - run: $CXX --version - - name: Build - run: $CXX source/cppfront.cpp $CXXFLAGS -o cppfront diff --git a/.github/workflows/build-test-pipeline.yaml b/.github/workflows/build-test-pipeline.yaml new file mode 100644 index 0000000000..316026cdb5 --- /dev/null +++ b/.github/workflows/build-test-pipeline.yaml @@ -0,0 +1,57 @@ +name: Build Test Pipeline + +on: + pull_request: + branches-ignore: + - docs + push: + branches-ignore: + - docs + workflow_dispatch: + +jobs: + build-test-pipeline: + runs-on: ${{ matrix.runs-on }} + name: ${{ matrix.target }} + strategy: + fail-fast: false + matrix: + include: + - target: x64-linux-g++-c++20 + runs-on: ubuntu-latest + compiler-path: /usr/bin/g++ + compiler-flags: -std=c++20 -Wall -Wextra -pedantic -Werror -O2 '-I{1}' -o '{2}' '{0}' + - target: x64-windows-msvc-c++latest + runs-on: windows-latest + compiler-path: cl.exe + compiler-flags: /std:c++latest /MD /EHsc /experimental:module /W4 /WX /O2 /I '{1}' '{0}' /link '/out:{2}' + # - target: x64-macos-clang-c++20 + # runs-on: macos-latest-large + # compiler-path: /usr/bin/clang++ + # compiler-flags: -std=c++20 -Wall -Wextra -pedantic -Werror -I{include_dir} -o {exe_out} {source_file} + # - target: arm64-macos-clang-c++20 + # runs-on: macos-latest + # compiler-path: /usr/bin/clang++ + # compiler-flags: -std=c++20 -Wall -Wextra -pedantic -Werror -I{include_dir} -o {exe_out} {source_file} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Activate MSVC dev environment (Windows only) + if: startsWith(matrix.runs-on, 'windows') + uses: ilammy/msvc-dev-cmd@v1 + + - name: Build cppfront + run: ${{ matrix.compiler-path }} ${{ format(matrix.compiler-flags, 'source/cppfront.cpp', './include/', 'cppfront.exe') }} + + - name: Run passthrough test + run: echo TODO + + - name: Transpile regression-runner + run: ./cppfront.exe -in -cwd test/ regression-runner.cpp2 + + - name: Build regression-runner + run: ${{ matrix.compiler-path }} ${{ format(matrix.compiler-flags, 'test/regression-runner.cpp', './include/', 'regression-runner.exe') }} + + - name: Run regression-runner w/ directory (All tests) + run: ./regression-runner.exe ${{ matrix.target }} ./cppfront.exe ./test/regression ${{ matrix.compiler-path }} ./include/ ${{ format(matrix.compiler-flags, '{source_file}', '{include_dir}', '{exe_out}') }} diff --git a/.github/workflows/regression-tests.yml b/.github/workflows/regression-tests.yml deleted file mode 100644 index 8aa4037e86..0000000000 --- a/.github/workflows/regression-tests.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Regression tests - -on: - pull_request: - branches-ignore: - - docs - push: - branches-ignore: - - docs - workflow_dispatch: - -jobs: - regression-tests: - name: ${{ matrix.shortosname }} | ${{ matrix.compiler }} | ${{ matrix.cxx_std }} | ${{ matrix.stdlib }} | ${{ matrix.os }} - runs-on: ${{ matrix.os }} - env: - CXX: ${{ matrix.compiler }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - shortosname: [ubu-24] - compiler: [g++-14, g++-13] - cxx_std: [c++2b] - stdlib: [libstdc++] - include: - - os: ubuntu-20.04 - shortosname: ubu-20 - compiler: g++-10 - cxx_std: c++20 - stdlib: libstdc++ - - os: ubuntu-24.04 - shortosname: ubu-24 - compiler: clang++-18 - cxx_std: c++20 - stdlib: libstdc++ - - os: ubuntu-24.04 - shortosname: ubu-24 - compiler: clang++-18 - cxx_std: c++23 - stdlib: libc++-18-dev - - os: ubuntu-22.04 - shortosname: ubu-22 - compiler: clang++-15 - cxx_std: c++20 - stdlib: libstdc++ - - os: ubuntu-22.04 - shortosname: ubu-22 - compiler: clang++-15 - cxx_std: c++20 - stdlib: libc++-15-dev - - os: ubuntu-20.04 - shortosname: ubu-20 - compiler: clang++-12 - cxx_std: c++20 - stdlib: libstdc++ - - os: macos-14 - shortosname: mac-14 - compiler: clang++ - cxx_std: c++2b - stdlib: default - - os: macos-13 - shortosname: mac-13 - compiler: clang++ - cxx_std: c++2b - stdlib: default - - os: macos-13 - shortosname: mac-13 - compiler: clang++-15 - cxx_std: c++2b - stdlib: default - - os: windows-2022 - shortosname: win-22 - compiler: cl.exe - cxx_std: c++latest - stdlib: default - - os: windows-2022 - shortosname: win-22 - compiler: cl.exe - cxx_std: c++20 - stdlib: default - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Prepare compilers - if: matrix.os == 'macos-13' - run: | - sudo xcode-select --switch /Applications/Xcode_14.3.1.app - sudo ln -s "$(brew --prefix llvm@15)/bin/clang" /usr/local/bin/clang++-15 - - - name: Run regression tests - Linux and macOS version - if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos') - run: | - cd regression-tests - bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }} - - - name: Run regression tests - Windows version - if: startsWith(matrix.os, 'windows') - run: | - "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^ - git config --local core.autocrlf false && ^ - cd regression-tests && ^ - bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }} - shell: cmd - - - name: Upload patch - if: success() || failure() - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch - path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch - if-no-files-found: ignore - - aggregate-results: - needs: regression-tests - if: success() || failure() - runs-on: ubuntu-latest - steps: - - name: Download all patches - uses: actions/download-artifact@v4 - with: - path: downloaded-results - - - name: Prepare result files - id: prepare_files - run: | - mkdir aggregated-results - echo "Flattening file hierarchy" - find . -type f -wholename "./downloaded-results*" -exec mv {} aggregated-results \; - patch_count=$(ls aggregated-results 2>/dev/null | wc -l) - echo "patch_count=${patch_count}" >> $GITHUB_OUTPUT - - - name: Upload aggregated results - if: steps.prepare_files.outputs.patch_count != '0' - uses: actions/upload-artifact@v4 - with: - name: aggregated-results - path: aggregated-results - if-no-files-found: ignore diff --git a/account_tests.py b/account_tests.py new file mode 100644 index 0000000000..86ad77cdf9 --- /dev/null +++ b/account_tests.py @@ -0,0 +1,155 @@ +import pprint +from typing import Dict +from pathlib import Path +from dataclasses import dataclass, field + +TARGETS_MAPPING: Dict[str, str] = { + "apple-clang-14-c++2b" : "TBD01", + "apple-clang-15-c++2b" : "TBD02", + "clang-12-c++20" : "TBD03", + "clang-15-c++20" : "TBD04", + "clang-15-c++20-libcpp": "TBD05", + "clang-18-c++20" : "TBD06", + "clang-18-c++23-libcpp": "TBD07", + "gcc-10-c++20" : "TBD08", + "gcc-13-c++2b" : "TBD09", + "gcc-14-c++2b" : "TBD10", + "msvc-2022-c++20" : "TBD11", + "msvc-2022-c++latest" : "TBD12", +} + +@dataclass +class TestProp: + has_lowered_output: bool + has_cppfront_output: bool + targets_with_cpp_exec: Dict[str, int] = field(default_factory=dict) + targets_with_cpp_output: Dict[str, int] = field(default_factory=dict) + +def main(): + if not Path("README.md").is_file(): + print("CWD should be root of the cppfront repo") + exit() + + p = Path(".") + trp = p / "regression-tests" / "test-results" + + files_accounted_for = [] + def try_account_for(pp): + if pp.is_file(): + files_accounted_for.append(pp) + return True + return False + + # Account for cppfront version + afcftp = try_account_for(trp / "version") + assert afcftp + + # Account for target compiler versions + for target in TARGETS_MAPPING.keys(): + compiler_prefix = "clang" + if "gcc" in target: + compiler_prefix = "gcc" + if "msvc" in target: + compiler_prefix = "MSVC" + afcvp = try_account_for(trp / target / (compiler_prefix + "-version.output")) + assert afcvp + + # Account for extraneous files (disable once fixed) + if True: + for efs in ( + "regression-tests/test-results/clang-18-c++20/pure2-bugfix-for-ufcs-arguments.cpp copy.execution", + "regression-tests/test-results/clang-18-c++23-libcpp/pure2-bugfix-for-ufcs-arguments.cpp copy.execution", + "regression-tests/test-results/apple-clang-14-c++2b/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/apple-clang-15-c++2b/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/clang-15-c++20-libcpp/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/clang-15-c++20/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/clang-18-c++20/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/clang-18-c++23-libcpp/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/gcc-13-c++2b/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/msvc-2022-c++20/pure2-regex_13_posessive_modifier.cpp.execution", + "regression-tests/test-results/msvc-2022-c++20/pure2-regex_13_posessive_modifier.cpp.output", + ): + afefp = try_account_for(Path(efs)) + assert afefp + + removable_msvc_files_count = 0 + tests: Dict[str, TestProp] = {} + for tp in p.glob("regression-tests/*.cpp2"): + aftp = try_account_for(tp) + assert aftp + + name = tp.name[:-5] + cppfront_lowered_output = trp / (name + ".cpp") + cppfront_output = trp / (name + ".cpp2.output") + + assert name not in tests + t = tests[name] = TestProp( + has_lowered_output=try_account_for(cppfront_lowered_output), + has_cppfront_output=try_account_for(cppfront_output), + ) + + for target in TARGETS_MAPPING.keys(): + cep = trp / target / (name + ".cpp.execution") + if try_account_for(cep): + t.targets_with_cpp_exec[target] = hash(open(cep).read()) + cop = trp / target / (name + ".cpp.output") + if try_account_for(cop): + # t.targets_with_cpp_output[target] = hash(open(cop).read()) + fcontent = open(cop).read() + if fcontent.strip() != (name + ".cpp"): + t.targets_with_cpp_output[target] = hash(fcontent) + else: + removable_msvc_files_count += 1 + + files_accounted_for_set = set([f for f in files_accounted_for]) + assert len(files_accounted_for_set) == len(files_accounted_for) + + all_files = set(fp for fp in p.glob("regression-tests/**/*") if not fp.is_dir()) + + redundant_file_count = len(all_files) - len(files_accounted_for) + + print(f"MSVC redundant removable files: {removable_msvc_files_count}\n") + redundant_file_count += removable_msvc_files_count + + print(f"Files unnacounted for ({redundant_file_count}):") + for f in sorted(all_files - files_accounted_for_set): + print(f"\t{f}") + print() + + print("All tests and their redundancy count:") + for k,v in sorted(tests.items()): + rk1 = _count_redundant_keys(v.targets_with_cpp_exec) + rk2 = _count_redundant_keys(v.targets_with_cpp_output) + redundant_file_count += rk1 + rk2 + + print(k) + pprint.pp(v) + print(f"-> {rk1}, {rk2}") + print() + print(f"Total redundant files = {redundant_file_count}") + print() + + tests_with_compiler_output = 0 + print("Tests with compiler output:") + for k,v in sorted(tests.items()): + if len(v.targets_with_cpp_output) == 0: + continue + tests_with_compiler_output += 1 + + print(k) + pprint.pp(v) + print() + print(f"Total = {tests_with_compiler_output}") + print() + +def _count_redundant_keys(d): + counts = {} + for k,v in d.items(): + counts[v] = counts.get(v,0)+1 + total = 0 + for v in counts.values(): + total += v - 1 + return total + +if __name__ == "__main__": + main() diff --git a/test/launch_program.hpp b/test/launch_program.hpp new file mode 100644 index 0000000000..9af26f52db --- /dev/null +++ b/test/launch_program.hpp @@ -0,0 +1,187 @@ +#ifndef LAUNCH_PROGRAM_HPP +#define LAUNCH_PROGRAM_HPP +#include +#include +#include + +struct launch_result +{ + int exit_status; + std::string output; +}; + +auto launch_program( + std::filesystem::path const& work_dir, + std::filesystem::path const& exe, + std::vector const& args +) -> launch_result; + +#ifdef LAUNCH_PROGRAM_IMPLEMENTATION +#ifdef _WIN32 +#define _WIN32_LEAN_AND_MEAN +#include +#include +#include + +auto launch_program( + std::filesystem::path const& work_dir, + std::filesystem::path const& exe, + std::vector const& args +) -> launch_result { + auto make_pipe_handle_pair = []() -> std::pair { + SECURITY_ATTRIBUTES saAttr{sizeof(SECURITY_ATTRIBUTES), nullptr, true}; + HANDLE read_end = nullptr; + HANDLE write_end = nullptr; + if(!CreatePipe(&read_end, &write_end, &saAttr, 0)) { return {}; } + if(!SetHandleInformation(read_end, HANDLE_FLAG_INHERIT, 0)) { + CloseHandle(write_end); + CloseHandle(read_end); + return {}; + } + return {read_end, write_end}; + }; + + auto [stdout_read, stdout_write] = make_pipe_handle_pair(); + auto [stderr_read, stderr_write] = make_pipe_handle_pair(); + + PROCESS_INFORMATION piProcInfo; + ZeroMemory(&piProcInfo, sizeof(PROCESS_INFORMATION)); + + STARTUPINFOA siStartInfo; + ZeroMemory(&siStartInfo, sizeof(siStartInfo)); + siStartInfo.cb = sizeof(siStartInfo); + siStartInfo.hStdOutput = stdout_write; + siStartInfo.hStdError = stderr_write; + siStartInfo.dwFlags |= STARTF_USESTDHANDLES; + + std::string cl_args = exe.string(); + for(const auto& arg : args) { cl_args.append(1, ' ').append(arg); } + + if( + !CreateProcessA( + exe.string().c_str(), + cl_args.data(), + nullptr, + nullptr, + true, + 0, + nullptr, + work_dir.string().c_str(), + &siStartInfo, + &piProcInfo + ) + ) { + + CloseHandle(piProcInfo.hProcess); + CloseHandle(piProcInfo.hThread); + + CloseHandle(stdout_write); + CloseHandle(stdout_read); + + CloseHandle(stderr_write); + CloseHandle(stderr_read); + + return {-1, {}}; + } + + CloseHandle(piProcInfo.hThread); + + CloseHandle(stdout_write); + CloseHandle(stderr_write); + + auto read_from_pipe = [](HANDLE read_end) { + std::array buff; + std::string out; + + while(true) { + DWORD bytes_read; + if( + !ReadFile( + read_end, + buff.data(), + static_cast(buff.size()), + &bytes_read, + nullptr + ) || bytes_read == 0 + ) { break; } + out.append(buff.data(), bytes_read); + } + + CloseHandle(read_end); + return out; + }; + + std::string output = read_from_pipe(stdout_read); + output += read_from_pipe(stderr_read); + + DWORD exitCode{}; + GetExitCodeProcess(piProcInfo.hProcess, &exitCode); + CloseHandle(piProcInfo.hProcess); + + return {static_cast(exitCode), output}; +} +#else // POSIX +#include +#include +#include +#include + +auto launch_program( + std::filesystem::path const& work_dir, + std::filesystem::path const& exe, + std::vector const& args +) -> launch_result { + std::vector cl_args; + cl_args.emplace_back(exe.c_str()); + for(const auto& arg : args) { cl_args.emplace_back(arg.c_str()); } + cl_args.emplace_back(nullptr); + + static constexpr size_t READ_END = 0; + static constexpr size_t WRITE_END = 1; + + int pipefds[2]; + if(pipe(pipefds) != 0) + return {-1, {}}; + + auto const pid = fork(); + if(pid < 0) { + close(pipefds[READ_END]); close(pipefds[WRITE_END]); + return {-2, {}}; + } + + if(pid == 0) { // Child executes new program on work dir... + close(pipefds[READ_END]); // We wont ever read this pipe here. + + dup2(pipefds[WRITE_END], STDERR_FILENO); + dup2(pipefds[WRITE_END], STDOUT_FILENO); + + close(pipefds[WRITE_END]); // Let's be good citizens. + + if(chdir(work_dir.c_str()) != 0) + _exit(-3); + + execvp(exe.c_str(), const_cast(cl_args.data())); + _exit(-4); + } + // Parent continues execution... + close(pipefds[WRITE_END]); // We wont ever write this pipe here. + + std::string output; + { + std::array buff; + ssize_t r; + while((r = read(pipefds[READ_END], buff.data(), buff.size())) > 0) { + output.append(buff.data(), r); + } + } + + close(pipefds[READ_END]); + int wstatus; + waitpid(pid, &wstatus, 0); + + return {WEXITSTATUS(wstatus), output}; +} + +#endif // _WIN32 +#endif // LAUNCH_PROGRAM_IMPLEMENTATION +#endif // LAUNCH_PROGRAM_HPP diff --git a/test/regression-runner.cpp2 b/test/regression-runner.cpp2 new file mode 100644 index 0000000000..2a0528783f --- /dev/null +++ b/test/regression-runner.cpp2 @@ -0,0 +1,292 @@ +/* + +Expected usage: + +./regression-runner + + +*/ +#define LAUNCH_PROGRAM_IMPLEMENTATION +#include "launch_program.hpp" + +fs: namespace == std::filesystem; + +main: (args) -> int = { + min_args :== 5; + assert( + args.ssize()-1 >= min_args, + "Expected at least (min_args)$ arguments. Received (args.ssize()-1)$\n", + ); + + ctx: testing_context = ( + args[1], + fs::path(args[2]).canonical(), + fs::path(args[4]), // We trust that this exe is in PATH. + fs::path(args[5]).canonical(), + ); + path_to_test := fs::path(args[3]).canonical(); + (copy i := min_args + 1) while i < args.ssize() next i++ { + _ = ctx.unformatted_compiler_args.emplace_back(args[i]); + } + + assert( + ctx.cppfront.is_regular_file(), + "Path to cppfront executable must be a regular file\n", + ); + + ext :== ".cpp2"; + + exit_status := EXIT_SUCCESS; + process_test_result := :(test_full_path: fs::path, result: test_result) = { + std::cout << "(test_full_path.filename().string())$: (result)$\n"; + if !result.is_ok() { exit_status&$* = EXIT_FAILURE; } + }; + + if !path_to_test.is_directory() { + assert( + path_to_test.extension() == ext, + "For one-shot testing, the path must point to a (ext)$ file\n", + ); + tr := test_one(ctx, path_to_test); + process_test_result(path_to_test, tr); + } else { + results: std::vector>> = (); + // Collect all tests... + for : fs::directory_iterator = (path_to_test) do (entry) + if entry.path().extension() == ext { + results.emplace_back().first = entry.path(); + } + // Sort them by name (optional!) + std::sort( + results.begin(), + results.end(), + : (lhs, rhs) -> _ = lhs.first.filename() < rhs.first.filename(), + ); + // Launch them all (you better have a good scheduler!) + for results do (inout r) { + r.second = std::async(test_one, ctx, r.first); + } + // Wait each task and display its result + for results do (inout r) { + tr := r.second.get(); + process_test_result(r.first, tr); + } + } + + return exit_status; +} + +testing_context: @struct type = { + target: std::string_view; + cppfront: fs::path; + compiler: fs::path; + include_dir: fs::path; + unformatted_compiler_args: std::vector = (); + // override_files: bool; // TODO +} + +test_result: @flag_enum type = { + first_run; + lowered_output_mismatch; + cppfront_output_mismatch; + compiler_output_mismatch; + compiler_output_exists_when_it_shouldnt; // or viceversa + test_exe_output_mismatch; + test_exe_output_exists_when_it_shouldnt; // or viceversa + + is_ok: (this) -> bool == this == none; + is_first_run: (this) -> bool == has(first_run); +} + +test_one: ( + ctx: testing_context, + test_filepath: fs::path, +) -> test_result = { + ins: testing_instance = ( + test_filepath, + test_filepath.parent_path() / "results" / test_filepath.stem(), + fs::temp_directory_path() / "cppfront-regressions" / test_filepath.stem(), + nullptr, + ); + + if !ins.result_dir.exists() || ins.result_dir.is_empty() { + ins.result = test_result::first_run; + _ = ins.result_dir.create_directory(); + ins.output_dir = ins.result_dir&; + } else { + ins.output_dir = ins.work_dir&; + } + assert( + ins.result_dir.is_directory(), + "The result path ((ins.result_dir)$) must be a directory\n", + ); + assert(ins.output_dir != nullptr); + + _ = ins.work_dir.create_directories(); + _: finally = ( :() = { _ = ins.work_dir&$*.remove_all(); } ); + + source_fn := transpile(ctx, ins); + if source_fn.empty() { return ins.result; } + + executable_filepath := compile(ctx, ins, source_fn); + if executable_filepath.empty() { return ins.result; } + + execute(ins, executable_filepath); + + return ins.result; +} + +testing_instance: @struct type = { + test_filepath: fs::path; + result_dir: fs::path; + work_dir: fs::path; + output_dir: *const fs::path; + result: test_result = (); +} + +transpile: (ctx: testing_context, inout ins: testing_instance) -> fs::path = { + lowered_output_fn :== "00-lowered.cpp"; + lowered_output_filepath: const = ins.output_dir* / lowered_output_fn; + + cppfront_args: std::vector = ( + "-o", + lowered_output_filepath.string(), + ins.test_filepath.filename().string(), + ); + + launch_result := launch_program( + ins.test_filepath.parent_path(), + ctx.cppfront, + cppfront_args, + ); + + cppfront_output_fn :== "01-cppfront.output"; + cppfront_output_filepath: const = ins.output_dir* / cppfront_output_fn; + + (: std::ofstream = (cppfront_output_filepath)) << launch_result.output; + + if !ins.result.is_first_run() { + assert(ins.output_dir* != ins.result_dir); + + if (ins.result_dir / lowered_output_fn).read_file() != + lowered_output_filepath.read_file() { + ins.result |= test_result::lowered_output_mismatch; + } + + if (ins.result_dir / cppfront_output_fn).read_file() != + cppfront_output_filepath.read_file() { + ins.result |= test_result::cppfront_output_mismatch; + } + } + + if lowered_output_filepath.exists() { + return lowered_output_fn; + } else { + return (); + } +} + +compile: ( + ctx: testing_context, + inout ins: testing_instance, + source_fn: fs::path +) -> fs::path = { + compiler_exe_out_path: const = ins.work_dir / "test.exe"; + + // NOTE: We always want this to be run with result's path so compiler + // error output matches. + // TODO: Review since execution can continue with a wrong source file atm + source_file: const = ins.result_dir / source_fn; + + compiler_args: std::vector = (); + for ctx.unformatted_compiler_args do (uarg) { + arg := compiler_args.emplace_back(uarg)&; + arg*.replace_all("{source_file}", source_file.string()); + arg*.replace_all("{include_dir}", ctx.include_dir.string()); + arg*.replace_all("{exe_out}", compiler_exe_out_path.string()); + } + + launch_result := launch_program( + ins.work_dir, + ctx.compiler, + compiler_args, + ); + + compiler_output_fn: const = "02-(ctx.target)$-compiler.output"; + compiler_output_filepath: const = ins.output_dir* / compiler_output_fn; + + if !launch_result.output.empty() { + (: std::ofstream = (compiler_output_filepath)) << launch_result.output; + } + + if !ins.result.is_first_run() { + assert(ins.output_dir* != ins.result_dir); + + if !launch_result.output.empty() { + if (ins.result_dir / compiler_output_fn).read_file() != + compiler_output_filepath.read_file() { + ins.result |= test_result::compiler_output_mismatch; + } + } else if compiler_output_filepath.exists() { + ins.result |= test_result::compiler_output_exists_when_it_shouldnt; + } + } + + if compiler_exe_out_path.exists() { + return compiler_exe_out_path; + } else { + return (); + } +} + +execute: ( + inout ins: testing_instance, + executable_filepath: fs::path, +) = { + test_exe_output_fn :== "03-executable.output"; + test_exe_output_filepath: const = ins.output_dir* / test_exe_output_fn; + + cmd_result := launch_program( + ins.work_dir, + executable_filepath, + : std::vector = (), + ); + + if !cmd_result.output.empty() { + (: std::ofstream = (test_exe_output_filepath)) << cmd_result.output; + } + + if !ins.result.is_first_run() { + assert(ins.output_dir* != ins.result_dir); + + if !cmd_result.output.empty() { + if (ins.result_dir / test_exe_output_fn).read_file() != + test_exe_output_filepath.read_file() { + ins.result |= test_result::test_exe_output_mismatch; + } + } else if test_exe_output_filepath.exists() { + ins.result |= test_result::test_exe_output_exists_when_it_shouldnt; + } + } +} + +replace_all: ( + inout subject: std::string, + search: std::string_view, + replacement: std::string_view, +) = { + pos: std::string::size_type = 0; + while (pos = subject.find(search, pos)) != std::string::npos { + _ = subject.replace(pos, search.size(), replacement); + pos += replacement.size(); + } +} + +read_file: (fp: fs::path) -> std::string = { + out: std::string = (); + b: std::array = (); // TODO: Find a way to avoid this init? + bd := b.data(); + f: std::ifstream = (fp, std::ios::in | std::ios::binary); + while f.read(bd, b.size()) { _ = out.append(bd, 0, f.gcount()); } + return out; +} diff --git a/test/regression/results/.gitkeep b/test/regression/results/.gitkeep new file mode 100644 index 0000000000..e69de29bb2