Redesign Windows build (#293)

.github/workflows/windows.yml

@@ -12,136 +12,60 @@ on:
   pull_request:
 
 jobs:
-  build:
+  build-zip:
     # Even though jou.exe runs on windows, it is compiled on linux.
     # This is by far the easiest way to compile for Windows that I know of.
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Download LLVM installer
-        run: wget --no-verbose https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.1/LLVM-13.0.1-win64.exe
-      - name: Verify LLVM installer
-        run: |
-          ls -lh LLVM-13.0.1-win64.exe
-          if [ "$(sha256sum LLVM-13.0.1-win64.exe)" == "9d15be034d52ec57cfc97615634099604d88a54761649498daa7405983a7e12f  LLVM-13.0.1-win64.exe" ]; then
-            echo ok
-          else
-            echo "sha sum mismatch! something has changed!"
-            exit 1
-          fi
-      # Apparently the exe file is created with nsis installer and 7z can extract from it.
-      # Figured out by looking at source code of https://github.com/KyleMayes/install-llvm-action
-      # TODO: could we instead statically link the Jou compiler?
-      - name: Extract files from LLVM installer
-        run: |
-          files="lib/LLVM-C.lib"
-          for file in $(7z l LLVM-13.0.1-win64.exe | grep -o 'bin/.*\.dll'); do
-            case $file in
-              # To figure out which dll files I need, I deleted them one by one and ran
-              # the compiler again.
-              #
-              # Unfortunately you need to do this locally instead of relying on github
-              # actions, because github actions comes with lots of software and hence lots
-              # of DLL files preinstalled. I used a Windows VM with nothing installed.
-              bin/LLVM-C.dll | \
-              bin/msvcp140.dll | \
-              bin/ucrtbase.dll | \
-              bin/vcruntime140.dll | \
-              bin/vcruntime140_1.dll | \
-              bin/api-ms-win-*.dll)     # Not sure which of these we need and what each one does.
-                files="$files $file"
-                ;;
-              *)
-                echo "*** skip dll: $file ***"
-                ;;
-            esac
-          done
-          echo "Extracting $files"
-          7z x LLVM-13.0.1-win64.exe $files
-      - name: Download MinGW
-        run: wget --no-verbose https://github.com/niXman/mingw-builds-binaries/releases/download/12.2.0-rt_v10-rev1/x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z
-      - name: Verify MinGW
-        run: |
-          ls -lh x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z
-          if [ "$(sha256sum x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z)" == "774916c4403c5219f8af3a3ee3012de6c017c034895c2c92bc4de99895c2c924  x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z" ]; then
-            echo ok
-          else
-            echo "sha sum mismatch! something has changed!"
-            exit 1
-          fi
-      # Exctract only the parts needed for linking.
-      # Figured out by exctracting only gcc.exe and then other files
-      # until hello world links successfully.
-      #
-      # We could get rid of gcc.exe and take only ld.exe, but
-      # gcc.exe is convenient because ld wants a long and complicated
-      # list of parameters that gcc can figure out for us.
-      - name: Extract the linker from MinGW
-        run: >
-          7z x x86_64-12.2.0-release-win32-seh-rt_v10-rev1.7z
-          mingw64/bin/gcc.exe
-          mingw64/lib/gcc/x86_64-w64-mingw32/12.2.0/libgcc.a
-          mingw64/lib/gcc/x86_64-w64-mingw32/12.2.0/libgcc_eh.a
-          mingw64/libexec/gcc/x86_64-w64-mingw32/12.2.0/liblto_plugin.dll
-          mingw64/x86_64-w64-mingw32/bin/ld.exe
-          mingw64/x86_64-w64-mingw32/lib/
-          mingw64/licenses/
-      # Using gcc instead of clang, because gcc "just works".
-      # llvm-13-dev needed for the header files. They seem to be missing from LLVM windows installer.
-      - name: Install gcc and dependencies for cross-compiling
-        run: sudo apt update && sudo apt install -y llvm-13-dev gcc-mingw-w64-x86-64-win32 dos2unix
-      - run: CC=x86_64-w64-mingw32-gcc LDFLAGS=lib/LLVM-C.lib make
+      - run: source activate && ./windows_setup.sh
+        shell: bash
+      - run: source activate && mingw32-make
+        shell: bash
       - run: mkdir jou
       - name: Copy files
         # Please keep this list of files in sync with update.ps1
-        run: cp -rv stdlib doc examples mingw64 LICENSE jou.exe update.ps1 bin/*.dll jou
-      - name: Convert files to Windows-style CRLF line endings
-        run: unix2dos $(find jou -name '*.jou') $(find jou -name '*.md') jou/LICENSE
-      - run: zip -r jou.zip jou
+        run: cp -rv stdlib doc examples mingw64 LICENSE jou.exe update.ps1 jou
+        shell: bash
+      - name: Copy DLL files
+        shell: bash
+        run: |
+          ready=no
+          while [ $ready == no ]; do
+            ready=yes
+            for file in $(objdump -p jou/jou.exe jou/*.dll | grep 'DLL Name:' | cut -d: -f2 | grep -vEi 'kernel32.dll|msvcrt.dll|advapi32.dll|ole32.dll|shell32.dll'); do
+              if ! [ -f jou/$file ]; then
+                cp -v mingw64/bin/$file jou/
+                ready=no
+              fi
+            done
+          done
+      - name: Convert text files to Windows-style CRLF line endings
+        run: mingw64/bin/unix2dos $(find jou -name '*.jou') $(find jou -name '*.md') jou/LICENSE
+        shell: bash
+      - run: Compress-Archive -Path jou -DestinationPath jou.zip
       - uses: actions/upload-artifact@v3
         with:
           name: windows-zip
           path: jou.zip
 
-  codeblocks-project:
-    # Ensure that the codeblocks project contains all source files.
-    # It gets outdated easily when I create a new file on linux.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - run: |
-          diff -u --color=always <(ls src/* | sort) <(grep filename= jou.cbp | cut -d'"' -f2)
-
   test:
-    needs: build
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
         with:
-          path: repo
-      - uses: actions/download-artifact@v3
-        with:
-          name: windows-zip
-      - run: unzip jou.zip
-      # Add a space in the folder name to trigger bugs like #165
-      - run: mkdir "test dir"
-      - run: mv jou/* repo/tests repo/runtests.sh "test dir"
-        shell: bash
-      - run: cd "test dir" && ./jou.exe --verbose examples/hello.jou
+          # Add a space in the folder name to trigger bugs like #165
+          path: "test dir"
+      - run: cd "test dir" && ./windows_setup.sh
         shell: bash
-      - run: cd "test dir" && ./runtests.sh --verbose
+      - run: cd "test dir" && source activate && ./runtests.sh --verbose
         shell: bash
 
   compare-compilers:
-    needs: build
     runs-on: windows-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
-        with:
-          name: windows-zip
-      - run: unzip jou.zip
-      - run: mv compare_compilers.sh self_hosted tests jou
+      - run: source activate && ./windows_setup.sh
         shell: bash
-      - run: (cd jou && ./compare_compilers.sh)
+      - run: source activate && ./compare_compilers.sh
         shell: bash

.gitignore

@@ -1,24 +1,24 @@
 /obj/
 /tmp/
 /jou
-/jou.exe
-/*.dll
 /compile_flags.txt
-jou_compiled
 /config.h
+/libs
+/mingw64
+/mingw64.zip
+
+jou_compiled
+*.dll
+*.exe
 
 # Jou update files
 /jou_update_info.json
 /jou_update.zip
 /jou_update
 
-# this is a weird hack, see #103
-/llvm_headers/
-
-# codeblocks stuff
-/bin/
-/jou.depend
-/jou.layout
+# llvm_headers.zip once extracted
+/llvm/
+/llvm-c/
 
 # ide stuff
 /.vscode/

CONTRIBUTING.md

@@ -16,46 +16,40 @@ to help `clangd` find the LLVM header files.
 <details> <summary>64-bit Windows</summary>
 
 1. Download and install Git from [Git's website](https://git-scm.com/download/win) if you don't have it already.
-1. To install LLVM, download and run LLVM's GitHub releases:
-    [LLVM-13.0.1-win64.exe from here](https://github.com/llvm/llvm-project/releases/tag/llvmorg-13.0.1), or
-    [LLVM-11.1.0-win64.exe from here](https://github.com/llvm/llvm-project/releases/tag/llvmorg-11.1.0).
-    Check the "Add LLVM to the system PATH for all users" checkbox during the installation.
-2. Download and install codeblocks from one of the download sites linked on
-    [their official website](http://www.codeblocks.org/downloads/binaries/#imagesoswindows48pnglogo-microsoft-windows).
-    Make sure to download a version that comes with mingw,
-    such as `codeblocks-20.03mingw-setup.exe`.
-    In the setup, the "Standard" installation contains everything you need.
-4. Add `C:\Program Files\CodeBlocks\MinGW\bin` to the `PATH` environment variable through Control Panel.
-    Without this, CodeBlocks doesn't find `clang`, the C compiler that LLVM comes with that is used to compile the Jou compiler.
-    Let me know if you need more detailed instructions for this step.
-5. Clone the project with the command prompt:
+2. Open Git Bash from the start menu.
+    **You must use Git Bash** for running bash scripts such as `windows_setup.sh` and `runtests.sh`.
+3. Clone the project with the command prompt:
     ```
     cd Desktop
     git clone https://github.com/Akuli/jou
     ```
     You can put the project anywhere. The above command places it on the desktop.
-6. Open the `jou` folder that you cloned with Git.
-7. Right-click `llvm_headers.zip` and extract it.
-    You should end up with a folder named `llvm_headers` inside the `jou` folder.
-8. Start CodeBlocks. It will probably ask you what should be the default C compiler.
-    This doesn't really matter because Jou comes with configuration that overrides the default anyway.
-8. Open the CodeBlocks project (`jou.cbp` in the `jou` folder) with CodeBlocks.
-10. Click the build button (yellow gear) at top of CodeBlocks.
-    If everything succeeds, this creates `jou.exe`.
-    If something goes wrong, please create an issue on GitHub.
-11. Run a Jou program:
+4. Run a script that does the rest of the setup for you:
     ```
-    cd Desktop\jou
-    jou.exe examples\hello.jou
+    cd jou
+    ./windows_setup.sh
+    ```
+5. Compile Jou:
+    ```
+    source activate
+    mingw32-make
+    ```
+    The `source activate` command adds `C:\Users\YourName\Desktop\jou\mingw64\bin` to your PATH,
+    where `C:\Users\YourName\Desktop` is the folder where you cloned Jou.
+    If you don't want to run it every time you open a Git Bash window to work on Jou,
+    you can instead add it to your PATH permanently with Control Panel.
+6. Compile and run hello world:
+    ```
+    ./jou.exe examples/hello.jou
     ```
     You should see `Hello World` printed.
-
-If CodeBlocks won't start and complains about a missing file `api-ms-win-crt-string-l1-1-0.dll`,
-make sure that LLVM is installed and you remembered to add it to `PATH`.
-LLVM conveniently comes with a DLL file that CodeBlocks developers apparently forgot to include.
-
-CodeBlocks doesn't have a dark theme by default.
-You can install a dark theme from e.g. [https://github.com/virtualmanu/Codeblocks-Themes](https://github.com/virtualmanu/Codeblocks-Themes).
+    If you instead get errors about missing DLL files, run `source activate` first.
+    The Jou compiler depends on DLLs in `mingw64\bin`,
+    so `mingw64\bin` must be in PATH when running it.
+7. Run tests:
+    ```
+    ./runtests.sh
+    ```
 
 </details>
 
@@ -100,7 +94,7 @@ $ ./runtests.sh
 ```
 
 This command does a few things:
-- If not on Windows, it compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled. (On Windows you need to use the build button in CodeBlocks to compile.)
+- I compiles the Jou compiler if you have changed something in `src/` since the last time it was compiled.
 - It runs all Jou files in `examples/` and `tests/`. To speed things up, it runs two files in parallel.
 - It ensures that the Jou files output what is expected.
 

Makefile

@@ -1,35 +1,13 @@
-LLVM_CONFIG ?= $(shell which llvm-config-13 || which llvm-config-11)
-
-SRC := $(wildcard src/*.c)
-
-ifeq ($(CC),cc)
-	# default c compiler --> use clang
-	CC := $(shell $(LLVM_CONFIG) --bindir)/clang
-endif
 CFLAGS += -Wall -Wextra -Wpedantic
 CFLAGS += -Werror=switch -Werror=implicit-function-declaration -Werror=incompatible-pointer-types -Werror=implicit-fallthrough
 CFLAGS += -std=c11
 CFLAGS += -g
-CFLAGS += $(shell $(LLVM_CONFIG) --cflags)
-LDFLAGS ?= $(shell $(LLVM_CONFIG) --ldflags --libs)
-
-all: jou compile_flags.txt
 
-# point clangd to the right include folder so i don't get red squiggles in my editor
-compile_flags.txt:
-	echo "-I$(shell $(LLVM_CONFIG) --includedir)"  > compile_flags.txt
-
-config.h:
-	echo "// auto-generated by Makefile" > config.h
-	echo "#define JOU_CLANG_PATH \"$(shell $(LLVM_CONFIG) --bindir)/clang\"" >> config.h
-
-obj/%.o: src/%.c $(wildcard src/*.h) config.h
-	mkdir -vp obj && $(CC) -c $(CFLAGS) $< -o $@
-
-jou: $(SRC:src/%.c=obj/%.o)
-	$(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+SRC := $(wildcard src/*.c)
+OBJ := $(SRC:src/%.c=obj/%.o)
 
-.PHONY: clean
-clean:
-	rm -rvf obj jou jou.exe tmp config.h compile_flags.txt
-	find -name jou_compiled -print -exec rm -rf '{}' +
+ifneq (,$(findstring Windows,$(OS)))
+	include Makefile.windows
+else
+	include Makefile.posix
+endif

Makefile.posix

@@ -0,0 +1,29 @@
+LLVM_CONFIG ?= $(shell which llvm-config-13 || which llvm-config-11)
+CFLAGS += $(shell $(LLVM_CONFIG) --cflags)
+LDFLAGS ?= $(shell $(LLVM_CONFIG) --ldflags --libs)
+
+ifeq ($(CC),cc)
+	# default c compiler --> use clang
+	CC := $(shell $(LLVM_CONFIG) --bindir)/clang
+endif
+
+all: jou compile_flags.txt
+
+# point clangd to the right include folder so i don't get red squiggles in my editor
+compile_flags.txt:
+	echo "-I$(shell $(LLVM_CONFIG) --includedir)"  > compile_flags.txt
+
+config.h:
+	echo "// auto-generated by Makefile" > config.h
+	echo "#define JOU_CLANG_PATH \"$(shell $(LLVM_CONFIG) --bindir)/clang\"" >> config.h
+
+obj/%.o: src/%.c $(wildcard src/*.h) config.h
+	mkdir -vp obj && $(CC) -c $(CFLAGS) $< -o $@
+
+jou: $(SRC:src/%.c=obj/%.o)
+	$(CC) $(CFLAGS) $^ -o $@ $(LDFLAGS)
+
+.PHONY: clean
+clean:
+	rm -rvf obj jou jou.exe tmp config.h compile_flags.txt
+	find -name jou_compiled -print -exec rm -rf '{}' +

Makefile.windows

@@ -0,0 +1,33 @@
+# assume llvm_headers.zip has been extracted
+CFLAGS += -I.
+
+# .a files generated in windows_setup.sh
+LDFLAGS += $(wildcard libs/lib*.a)
+
+ifeq ($(CC),cc)
+	# default c compiler --> use clang
+	CC := mingw64/bin/clang.exe
+endif
+
+# clang version in mingw doesn't seem as mature as what I have on linux...
+# it shows a lot of unnecssary/dumb warnings by default
+CFLAGS += -Wno-return-type -Wno-uninitialized -Wno-implicit-fallthrough
+
+all: jou.exe compile_flags.txt
+
+# point clangd to the right include folder so i don't get red squiggles in my editor
+compile_flags.txt:
+	echo -I$(CURDIR) > compile_flags.txt
+
+obj/%.o: src/%.c $(wildcard src/*.h)
+	(mkdir obj 2>NUL || cd .) && $(CC) -c $(CFLAGS) $< -o $@
+
+jou.exe: $(OBJ)
+	$(CC) $(CFLAGS) $(OBJ) -o $@ $(LDFLAGS)
+
+.PHONY: clean
+clean:
+	rmdir /s /q obj tmp 2>NUL
+	del /q jou.exe compile_flags.txt 2>NUL
+# TODO: ideally we would also delete jou_compiled directories, but not sure how:
+#	find -name jou_compiled -print -exec rm -rf '{}' +