From 3f3a893ef0e3eb949d16bb8cd964961a226a9d48 Mon Sep 17 00:00:00 2001 From: Akuli Date: Sat, 4 Mar 2023 16:42:43 +0200 Subject: [PATCH] Redesign Windows build (#293) --- .github/workflows/windows.yml | 138 ++++++++-------------------------- .gitignore | 20 ++--- CONTRIBUTING.md | 60 +++++++-------- Makefile | 36 ++------- Makefile.posix | 29 +++++++ Makefile.windows | 33 ++++++++ activate | 15 ++++ compare_compilers.sh | 3 + jou.cbp | 94 ----------------------- runtests.sh | 5 +- windows_setup.sh | 92 +++++++++++++++++++++++ 11 files changed, 251 insertions(+), 274 deletions(-) create mode 100644 Makefile.posix create mode 100644 Makefile.windows create mode 100644 activate delete mode 100644 jou.cbp create mode 100755 windows_setup.sh diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 296a0d50..2c78fa2b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -12,136 +12,60 @@ on: pull_request: jobs: - build: + build-zip: # Even though jou.exe runs on windows, it is compiled on linux. # This is by far the easiest way to compile for Windows that I know of. - runs-on: ubuntu-latest + runs-on: windows-latest steps: - uses: actions/checkout@v3 - - name: Download LLVM installer - run: wget --no-verbose https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/LLVM-13.0.1-win64.exe - - name: Verify LLVM installer - run: | - ls -lh LLVM-13.0.1-win64.exe - if [ "$(sha256sum LLVM-13.0.1-win64.exe)" == "9d15be034d52ec57cfc97615634099604d88a54761649498daa7405983a7e12f LLVM-13.0.1-win64.exe" ]; then - echo ok - else - echo "sha sum mismatch! something has changed!" - exit 1 - fi - # Apparently the exe file is created with nsis installer and 7z can extract from it. - # Figured out by looking at source code of https://github.com/KyleMayes/install-llvm-action - # TODO: could we instead statically link the Jou compiler? - - name: Extract files from LLVM installer - run: | - files="lib/LLVM-C.lib" - for file in $(7z l LLVM-13.0.1-win64.exe | grep -o 'bin/.*\.dll'); do - case $file in - # To figure out which dll files I need, I deleted them one by one and ran - # the compiler again. - # - # Unfortunately you need to do this locally instead of relying on github - # actions, because github actions comes with lots of software and hence lots - # of DLL files preinstalled. I used a Windows VM with nothing installed. - bin/LLVM-C.dll | \ - bin/msvcp140.dll | \ - bin/ucrtbase.dll | \ - bin/vcruntime140.dll | \ - bin/vcruntime140_1.dll | \ - bin/api-ms-win-*.dll) # Not sure which of these we need and what each one does. - files="$files $file" - ;; - *) - echo "*** skip dll: $file ***" - ;; - esac - done - echo "Extracting $files" - 7z x LLVM-13.0.1-win64.exe $files - - name: Download MinGW - run: wget --no-verbose https://github.com/niXman/mingw-builds-binaries/releases/download/12.2.0-rt_v10-rev1/x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z - - name: Verify MinGW - run: | - ls -lh x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z - if [ "$(sha256sum x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z)" == "774916c4403c5219f8af3a3ee3012de6c017c034895c2c92bc4de99895c2c924 x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z" ]; then - echo ok - else - echo "sha sum mismatch! something has changed!" - exit 1 - fi - # Exctract only the parts needed for linking. - # Figured out by exctracting only gcc.exe and then other files - # until hello world links successfully. - # - # We could get rid of gcc.exe and take only ld.exe, but - # gcc.exe is convenient because ld wants a long and complicated - # list of parameters that gcc can figure out for us. - - name: Extract the linker from MinGW - run: > - 7z x x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z - mingw64/bin/gcc.exe - mingw64/lib/gcc/x86_64-w64-mingw32/12.2.0/libgcc.a - mingw64/lib/gcc/x86_64-w64-mingw32/12.2.0/libgcc_eh.a - mingw64/libexec/gcc/x86_64-w64-mingw32/12.2.0/liblto_plugin.dll - mingw64/x86_64-w64-mingw32/bin/ld.exe - mingw64/x86_64-w64-mingw32/lib/ - mingw64/licenses/ - # Using gcc instead of clang, because gcc "just works". - # llvm-13-dev needed for the header files. They seem to be missing from LLVM windows installer. - - name: Install gcc and dependencies for cross-compiling - run: sudo apt update && sudo apt install -y llvm-13-dev gcc-mingw-w64-x86-64-win32 dos2unix - - run: CC=x86_64-w64-mingw32-gcc LDFLAGS=lib/LLVM-C.lib make + - run: source activate && ./windows_setup.sh + shell: bash + - run: source activate && mingw32-make + shell: bash - run: mkdir jou - name: Copy files # Please keep this list of files in sync with update.ps1 - run: cp -rv stdlib doc examples mingw64 LICENSE jou.exe update.ps1 bin/*.dll jou - - name: Convert files to Windows-style CRLF line endings - run: unix2dos $(find jou -name '*.jou') $(find jou -name '*.md') jou/LICENSE - - run: zip -r jou.zip jou + run: cp -rv stdlib doc examples mingw64 LICENSE jou.exe update.ps1 jou + shell: bash + - name: Copy DLL files + shell: bash + run: | + ready=no + while [ $ready == no ]; do + ready=yes + for file in $(objdump -p jou/jou.exe jou/*.dll | grep 'DLL Name:' | cut -d: -f2 | grep -vEi 'kernel32.dll|msvcrt.dll|advapi32.dll|ole32.dll|shell32.dll'); do + if ! [ -f jou/$file ]; then + cp -v mingw64/bin/$file jou/ + ready=no + fi + done + done + - name: Convert text files to Windows-style CRLF line endings + run: mingw64/bin/unix2dos $(find jou -name '*.jou') $(find jou -name '*.md') jou/LICENSE + shell: bash + - run: Compress-Archive -Path jou -DestinationPath jou.zip - uses: actions/upload-artifact@v3 with: name: windows-zip path: jou.zip - codeblocks-project: - # Ensure that the codeblocks project contains all source files. - # It gets outdated easily when I create a new file on linux. - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - run: | - diff -u --color=always <(ls src/* | sort) <(grep filename= jou.cbp | cut -d'"' -f2) - test: - needs: build runs-on: windows-latest steps: - uses: actions/checkout@v3 with: - path: repo - - uses: actions/download-artifact@v3 - with: - name: windows-zip - - run: unzip jou.zip - # Add a space in the folder name to trigger bugs like #165 - - run: mkdir "test dir" - - run: mv jou/* repo/tests repo/runtests.sh "test dir" - shell: bash - - run: cd "test dir" && ./jou.exe --verbose examples/hello.jou + # Add a space in the folder name to trigger bugs like #165 + path: "test dir" + - run: cd "test dir" && ./windows_setup.sh shell: bash - - run: cd "test dir" && ./runtests.sh --verbose + - run: cd "test dir" && source activate && ./runtests.sh --verbose shell: bash compare-compilers: - needs: build runs-on: windows-latest steps: - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: windows-zip - - run: unzip jou.zip - - run: mv compare_compilers.sh self_hosted tests jou + - run: source activate && ./windows_setup.sh shell: bash - - run: (cd jou && ./compare_compilers.sh) + - run: source activate && ./compare_compilers.sh shell: bash diff --git a/.gitignore b/.gitignore index c59f8cc5..3ff9786e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,24 @@ /obj/ /tmp/ /jou -/jou.exe -/*.dll /compile_flags.txt -jou_compiled /config.h +/libs +/mingw64 +/mingw64.zip + +jou_compiled +*.dll +*.exe # Jou update files /jou_update_info.json /jou_update.zip /jou_update -# this is a weird hack, see #103 -/llvm_headers/ - -# codeblocks stuff -/bin/ -/jou.depend -/jou.layout +# llvm_headers.zip once extracted +/llvm/ +/llvm-c/ # ide stuff /.vscode/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81429c77..756bc185 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,46 +16,40 @@ to help `clangd` find the LLVM header files.
64-bit Windows 1. Download and install Git from [Git's website](https://git-scm.com/download/win) if you don't have it already. -1. To install LLVM, download and run LLVM's GitHub releases: - [LLVM-13.0.1-win64.exe from here](https://github.com/llvm/llvm-project/releases/tag/llvmorg-13.0.1), or - [LLVM-11.1.0-win64.exe from here](https://github.com/llvm/llvm-project/releases/tag/llvmorg-11.1.0). - Check the "Add LLVM to the system PATH for all users" checkbox during the installation. -2. Download and install codeblocks from one of the download sites linked on - [their official website](http://www.codeblocks.org/downloads/binaries/#imagesoswindows48pnglogo-microsoft-windows). - Make sure to download a version that comes with mingw, - such as `codeblocks-20.03mingw-setup.exe`. - In the setup, the "Standard" installation contains everything you need. -4. Add `C:\Program Files\CodeBlocks\MinGW\bin` to the `PATH` environment variable through Control Panel. - Without this, CodeBlocks doesn't find `clang`, the C compiler that LLVM comes with that is used to compile the Jou compiler. - Let me know if you need more detailed instructions for this step. -5. Clone the project with the command prompt: +2. Open Git Bash from the start menu. + **You must use Git Bash** for running bash scripts such as `windows_setup.sh` and `runtests.sh`. +3. Clone the project with the command prompt: ``` cd Desktop git clone https://github.com/Akuli/jou ``` You can put the project anywhere. The above command places it on the desktop. -6. Open the `jou` folder that you cloned with Git. -7. Right-click `llvm_headers.zip` and extract it. - You should end up with a folder named `llvm_headers` inside the `jou` folder. -8. Start CodeBlocks. It will probably ask you what should be the default C compiler. - This doesn't really matter because Jou comes with configuration that overrides the default anyway. -8. Open the CodeBlocks project (`jou.cbp` in the `jou` folder) with CodeBlocks. -10. Click the build button (yellow gear) at top of CodeBlocks. - If everything succeeds, this creates `jou.exe`. - If something goes wrong, please create an issue on GitHub. -11. Run a Jou program: +4. Run a script that does the rest of the setup for you: ``` - cd Desktop\jou - jou.exe examples\hello.jou + cd jou + ./windows_setup.sh + ``` +5. Compile Jou: + ``` + source activate + mingw32-make + ``` + The `source activate` command adds `C:\Users\YourName\Desktop\jou\mingw64\bin` to your PATH, + where `C:\Users\YourName\Desktop` is the folder where you cloned Jou. + If you don't want to run it every time you open a Git Bash window to work on Jou, + you can instead add it to your PATH permanently with Control Panel. +6. Compile and run hello world: + ``` + ./jou.exe examples/hello.jou ``` You should see `Hello World` printed. - -If CodeBlocks won't start and complains about a missing file `api-ms-win-crt-string-l1-1-0.dll`, -make sure that LLVM is installed and you remembered to add it to `PATH`. -LLVM conveniently comes with a DLL file that CodeBlocks developers apparently forgot to include. - -CodeBlocks doesn't have a dark theme by default. -You can install a dark theme from e.g. [https://github.com/virtualmanu/Codeblocks-Themes](https://github.com/virtualmanu/Codeblocks-Themes). + If you instead get errors about missing DLL files, run `source activate` first. + The Jou compiler depends on DLLs in `mingw64\bin`, + so `mingw64\bin` must be in PATH when running it. +7. Run tests: + ``` + ./runtests.sh + ```
@@ -100,7 +94,7 @@ $ ./runtests.sh ``` This command does a few things: -- If not on Windows, it compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. (On Windows you need to use the build button in CodeBlocks to compile.) +- I compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. - It runs all Jou files in `examples/` and `tests/`. To speed things up, it runs two files in parallel. - It ensures that the Jou files output what is expected. diff --git a/Makefile b/Makefile index 30c83a0a..ba75db73 100644 --- a/Makefile +++ b/Makefile @@ -1,35 +1,13 @@ -LLVM_CONFIG ?= $(shell which llvm-config-13 || which llvm-config-11) - -SRC := $(wildcard src/*.c) - -ifeq ($(CC),cc) - # default c compiler --> use clang - CC := $(shell $(LLVM_CONFIG) --bindir)/clang -endif CFLAGS += -Wall -Wextra -Wpedantic CFLAGS += -Werror=switch -Werror=implicit-function-declaration -Werror=incompatible-pointer-types -Werror=implicit-fallthrough CFLAGS += -std=c11 CFLAGS += -g -CFLAGS += $(shell $(LLVM_CONFIG) --cflags) -LDFLAGS ?= $(shell $(LLVM_CONFIG) --ldflags --libs) - -all: jou compile_flags.txt -# point clangd to the right include folder so i don't get red squiggles in my editor -compile_flags.txt: - echo "-I$(shell $(LLVM_CONFIG) --includedir)" > compile_flags.txt - -config.h: - echo "// auto-generated by Makefile" > config.h - echo "#define JOU_CLANG_PATH \"$(shell $(LLVM_CONFIG) --bindir)/clang\"" >> config.h - -obj/%.o: src/%.c $(wildcard src/*.h) config.h - mkdir -vp obj && $(CC) -c $(CFLAGS) $< -o $@ - -jou: $(SRC:src/%.c=obj/%.o) - $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS) +SRC := $(wildcard src/*.c) +OBJ := $(SRC:src/%.c=obj/%.o) -.PHONY: clean -clean: - rm -rvf obj jou jou.exe tmp config.h compile_flags.txt - find -name jou_compiled -print -exec rm -rf '{}' + +ifneq (,$(findstring Windows,$(OS))) + include Makefile.windows +else + include Makefile.posix +endif diff --git a/Makefile.posix b/Makefile.posix new file mode 100644 index 00000000..2cdc34a9 --- /dev/null +++ b/Makefile.posix @@ -0,0 +1,29 @@ +LLVM_CONFIG ?= $(shell which llvm-config-13 || which llvm-config-11) +CFLAGS += $(shell $(LLVM_CONFIG) --cflags) +LDFLAGS ?= $(shell $(LLVM_CONFIG) --ldflags --libs) + +ifeq ($(CC),cc) + # default c compiler --> use clang + CC := $(shell $(LLVM_CONFIG) --bindir)/clang +endif + +all: jou compile_flags.txt + +# point clangd to the right include folder so i don't get red squiggles in my editor +compile_flags.txt: + echo "-I$(shell $(LLVM_CONFIG) --includedir)" > compile_flags.txt + +config.h: + echo "// auto-generated by Makefile" > config.h + echo "#define JOU_CLANG_PATH \"$(shell $(LLVM_CONFIG) --bindir)/clang\"" >> config.h + +obj/%.o: src/%.c $(wildcard src/*.h) config.h + mkdir -vp obj && $(CC) -c $(CFLAGS) $< -o $@ + +jou: $(SRC:src/%.c=obj/%.o) + $(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS) + +.PHONY: clean +clean: + rm -rvf obj jou jou.exe tmp config.h compile_flags.txt + find -name jou_compiled -print -exec rm -rf '{}' + diff --git a/Makefile.windows b/Makefile.windows new file mode 100644 index 00000000..b880d187 --- /dev/null +++ b/Makefile.windows @@ -0,0 +1,33 @@ +# assume llvm_headers.zip has been extracted +CFLAGS += -I. + +# .a files generated in windows_setup.sh +LDFLAGS += $(wildcard libs/lib*.a) + +ifeq ($(CC),cc) + # default c compiler --> use clang + CC := mingw64/bin/clang.exe +endif + +# clang version in mingw doesn't seem as mature as what I have on linux... +# it shows a lot of unnecssary/dumb warnings by default +CFLAGS += -Wno-return-type -Wno-uninitialized -Wno-implicit-fallthrough + +all: jou.exe compile_flags.txt + +# point clangd to the right include folder so i don't get red squiggles in my editor +compile_flags.txt: + echo -I$(CURDIR) > compile_flags.txt + +obj/%.o: src/%.c $(wildcard src/*.h) + (mkdir obj 2>NUL || cd .) && $(CC) -c $(CFLAGS) $< -o $@ + +jou.exe: $(OBJ) + $(CC) $(CFLAGS) $(OBJ) -o $@ $(LDFLAGS) + +.PHONY: clean +clean: + rmdir /s /q obj tmp 2>NUL + del /q jou.exe compile_flags.txt 2>NUL +# TODO: ideally we would also delete jou_compiled directories, but not sure how: +# find -name jou_compiled -print -exec rm -rf '{}' + diff --git a/activate b/activate new file mode 100644 index 00000000..2c3e8591 --- /dev/null +++ b/activate @@ -0,0 +1,15 @@ +if (echo "$PATH" | tr : '\n' | grep -qFx "$PWD/mingw64/bin") || [ "$WINEPATH" = "$PWD/mingw64/bin" ]; then + echo "Already activated." +else + PS1="(jou dev) $PS1" + if [[ "$OS" =~ Windows ]]; then + export PATH="$PWD/mingw64/bin:$PATH" + else + echo "The activate script is useful in two scenarios:" + echo " 1) You use Windows" + echo " 2) You don't use Windows, but you want to work on Windows support with wine" + echo "" + echo "Assuming scenario 2." + export WINEPATH="$PWD/mingw64/bin" + fi +fi diff --git a/compare_compilers.sh b/compare_compilers.sh index 082783a3..1a41cf17 100755 --- a/compare_compilers.sh +++ b/compare_compilers.sh @@ -15,8 +15,11 @@ fi if [[ "$OS" =~ Windows ]]; then dotexe=.exe + source activate + mingw32-make else dotexe= + make fi set -e diff --git a/jou.cbp b/jou.cbp deleted file mode 100644 index be02a9ca..00000000 --- a/jou.cbp +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - diff --git a/runtests.sh b/runtests.sh index f2527d24..8fb75c6c 100755 --- a/runtests.sh +++ b/runtests.sh @@ -45,7 +45,10 @@ if [ $valgrind = yes ]; then command_template="valgrind -q --leak-check=full --show-leak-kinds=all --suppressions=valgrind-suppressions.sup $command_template" fi -if ! [[ "$OS" =~ Windows ]]; then +if [[ "$OS" =~ Windows ]]; then + source activate + mingw32-make +else make fi diff --git a/windows_setup.sh b/windows_setup.sh new file mode 100755 index 00000000..db44b49e --- /dev/null +++ b/windows_setup.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# Run this script in Git Bash to get started with developing Jou on Windows. +# It downloads everything you need (except a text editor). + +set -e + +if [[ "$OS" =~ Windows ]]; then + run= +else + # Make this script work on linux, so that I can develop it without windows + echo "Wine will be used to run Windows commands." + run=wine +fi + +# Keep the size in the command below up to date if you update WinLibs: +if [ -d mingw64 ] && [ $(du -s mingw64 | cut -f1) -gt 900000 ]; then + echo "mingw64 has already been downloaded and extracted." +else + # The WinLibs version we use ships with LLVM 14, which is latest LLVM that Jou can use. + # This is due to opaque pointer types. Scroll down to "Version Support" here: https://llvm.org/docs/OpaquePointers.html + # All WinLibs versions and download links: https://winlibs.com/ + echo "Downloading mingw64 (WinLibs)..." + curl -L -o mingw64.zip https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-msvcrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64msvcrt-10.0.0-r3.zip + + echo "Verifying mingw64..." + if [ "$(sha256sum mingw64.zip | cut -d' ' -f1)" != "9ffef7f7a8dab893bd248085fa81a5a37ed6f775ae220ef673bea8806677836d" ]; then + echo "Verifying $filename failed." >&2 + exit 1 + fi + + echo "Extracting mingw64..." + rm -rf mingw64 + unzip -q mingw64.zip + rm mingw64.zip +fi + +# Blog post that explains how .a library is generated from dll file on Windows: +# https://dev.my-gate.net/2018/06/02/generate-a-def-file-from-a-dll/ +# +# A simple "grep -r LLVMCreatePassManager" reveals that only libLLVMCore.dll +# contains the functions we need, so we generate a corresponding .a file. +# There are also a few other files that I found similarly. +mkdir -vp libs +echo "Generating .a files (this can take a while)" +for name in \ + libLLVMCore libLLVMX86CodeGen libLLVMAnalysis libLLVMTarget \ + libLLVMipo libLLVMLinker libLTO libLLVMX86AsmParser \ + libLLVMX86Info libLLVMX86Desc +do + if [ -f libs/$name.a ]; then + echo " libs/$name.a has already been generated." + else + echo " Generating libs/$name.a..." + cd libs + $run ../mingw64/bin/gendef.exe ../mingw64/bin/$name.dll + $run ../mingw64/bin/dlltool.exe -d $name.def -l $name.a + rm $name.def + cd .. + echo " done" + fi +done + +# These headers are a bit of a hack. They have been taken from a different +# LLVM version, specifically, LLVM 13 compiled for Linux. We really only +# need the headers to declare the functions and structs and enums we use. +# +# But still, committing a zip file to Git just doesn't seem great... +if [ -d llvm ] && [ -d llvm-c ]; then + echo "LLVM headers have already been extracted." +else + echo "Extracting LLVM headers..." + rm -rf llvm llvm-c + unzip -q llvm_headers.zip +fi + +echo "" +echo "" +echo "" +echo "" +echo "Your Jou development environment is now ready." +echo "Next you can compile the Jou compiler and run hello world:" +echo "" +echo " source activate" +if [ "$run" = "" ]; then + echo " mingw32-make" + echo " ./jou.exe examples/hello.jou" +else + echo " $run mingw32-make" + echo " $run ./jou.exe examples/hello.jou" +fi +echo ""