Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor code to make it more amenable to cross compilation #150

Merged
merged 8 commits into from
Mar 4, 2024
Merged

Refactor code to make it more amenable to cross compilation #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8c46be1
Refactor code to make it more amenable to cross compilation
gbaraldi Nov 28, 2023
0e0f872
Fixup macos issues
gbaraldi Nov 29, 2023
0d537cd
Add wasm test and fixup macos
gbaraldi Nov 29, 2023
bf0740b
Whitespace
gbaraldi Nov 29, 2023
2bf7026
Add way to set compiler for cross target and also bump major version
gbaraldi Dec 5, 2023
ac2d56f
Bump Bumper :)
gbaraldi Dec 5, 2023
e3412cb
Prepare code for future endeavours into adding the GC
gbaraldi Dec 5, 2023
0eb680b
Merge branch 'master' into gb/refactor-code
brenhinkeller Mar 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Project.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name = "StaticCompiler"
uuid = "81625895-6c0f-48fc-b932-11a18313743c"
authors = ["Tom Short and contributors"]
version = "0.6.3"
version = "0.7.0"


[deps]
Clang_jll = "0ee61d77-7f21-5576-8119-9fcc46b10100"
Expand Down
120 changes: 70 additions & 50 deletions src/StaticCompiler.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@ using LLD_jll: lld
using StaticTools
using StaticTools: @symbolcall, @c_str, println
using Core: MethodTable

using Base:BinaryPlatforms.Platform, BinaryPlatforms.HostPlatform, BinaryPlatforms.arch, BinaryPlatforms.os_str, BinaryPlatforms.libc_str
using Base:BinaryPlatforms.platform_dlext
export load_function, compile_shlib, compile_executable
export native_code_llvm, native_code_typed, native_llvm_module, native_code_native
export static_code_llvm, static_code_typed, static_llvm_module, static_code_native
export @device_override, @print_and_throw
export StaticTarget

include("interpreter.jl")
include("target.jl")
Expand All @@ -32,6 +34,7 @@ compile_executable(f::Function, types::Tuple, path::String, [name::String=string
filename::String=name,
cflags=``, # Specify libraries you would like to link against, and other compiler options here
also_expose=[],
target::StaticTarget=StaticTarget(),
method_table=StaticCompiler.method_table,
kwargs...
)
Expand Down Expand Up @@ -96,16 +99,16 @@ Hello, world!
```
"""
function compile_executable(f::Function, types=(), path::String="./", name=fix_name(f);
also_expose=Tuple{Function, Tuple{DataType}}[],
also_expose=Tuple{Function, Tuple{DataType}}[], target::StaticTarget=StaticTarget(),
kwargs...)

compile_executable(vcat([(f, types)], also_expose), path, name; kwargs...)
compile_executable(vcat([(f, types)], also_expose), path, name; target, kwargs...)
end

function compile_executable(funcs::Union{Array,Tuple}, path::String="./", name=fix_name(first(first(funcs)));
filename = name,
demangle = true,
cflags = ``,
target::StaticTarget=StaticTarget(),
kwargs...
)

Expand All @@ -114,12 +117,12 @@ function compile_executable(funcs::Union{Array,Tuple}, path::String="./", name=f
isexecutableargtype = tt == Tuple{} || tt == Tuple{Int, Ptr{Ptr{UInt8}}}
isexecutableargtype || @warn "input type signature $types should be either `()` or `(Int, Ptr{Ptr{UInt8}})` for standard executables"

rt = last(only(native_code_typed(f, tt; kwargs...)))
rt = last(only(static_code_typed(f, tt; target, kwargs...)))
isconcretetype(rt) || error("`$f$types` did not infer to a concrete type. Got `$rt`")
nativetype = isprimitivetype(rt) || isa(rt, Ptr)
nativetype || @warn "Return type `$rt` of `$f$types` does not appear to be a native type. Consider returning only a single value of a native machine type (i.e., a single float, int/uint, bool, or pointer). \n\nIgnoring this warning may result in Undefined Behavior!"

generate_executable(funcs, path, name, filename; demangle, cflags, kwargs...)
generate_executable(funcs, path, name, filename; demangle, cflags, target, kwargs...)
joinpath(abspath(path), filename)
end

Expand All @@ -129,13 +132,15 @@ compile_shlib(f::Function, types::Tuple, [path::String="./"], [name::String=stri
filename::String=name,
cflags=``,
method_table=StaticCompiler.method_table,
target::StaticTarget=StaticTarget(),
kwargs...)

compile_shlib(funcs::Array, [path::String="./"];
filename="libfoo",
demangle=true,
cflags=``,
method_table=StaticCompiler.method_table,
target::StaticTarget=StaticTarget(),
kwargs...)
```
As `compile_executable`, but compiling to a standalone `.dylib`/`.so` shared library.
Expand Down Expand Up @@ -169,33 +174,35 @@ julia> ccall(("test", "test.dylib"), Float64, (Int64,), 100_000)
"""
function compile_shlib(f::Function, types=(), path::String="./", name=fix_name(f);
filename=name,
target::StaticTarget=StaticTarget(),
kwargs...
)
compile_shlib(((f, types),), path; filename, kwargs...)
compile_shlib(((f, types),), path; filename, target, kwargs...)
end
# As above, but taking an array of functions and returning a single shlib
function compile_shlib(funcs::Union{Array,Tuple}, path::String="./";
filename = "libfoo",
demangle = true,
cflags = ``,
target::StaticTarget=StaticTarget(),
kwargs...
)
for func in funcs
f, types = func
tt = Base.to_tuple_type(types)
isconcretetype(tt) || error("input type signature `$types` is not concrete")

rt = last(only(native_code_typed(f, tt)))
rt = last(only(static_code_typed(f, tt; target, kwargs...)))
isconcretetype(rt) || error("`$f$types` did not infer to a concrete type. Got `$rt`")
nativetype = isprimitivetype(rt) || isa(rt, Ptr)
nativetype || @warn "Return type `$rt` of `$f$types` does not appear to be a native type. Consider returning only a single value of a native machine type (i.e., a single float, int/uint, bool, or pointer). \n\nIgnoring this warning may result in Undefined Behavior!"
end

generate_shlib(funcs, true, path, filename; demangle, cflags, kwargs...)
generate_shlib(funcs, path, filename; demangle, cflags, target, kwargs...)

joinpath(abspath(path), filename * "." * Libdl.dlext)
end


"""
```julia
Expand Down Expand Up @@ -281,14 +288,18 @@ generate_executable(f, tt, args...; kwargs...) = generate_executable(((f, tt),),
function generate_executable(funcs::Union{Array,Tuple}, path=tempname(), name=fix_name(first(first(funcs))), filename=name;
demangle = true,
cflags = ``,
target::StaticTarget=StaticTarget(),
kwargs...
)
lib_path = joinpath(path, "$filename.$(Libdl.dlext)")
exec_path = joinpath(path, filename)
external = true
_, obj_path = generate_obj(funcs, external, path, filename; demangle, kwargs...)
_, obj_path = generate_obj(funcs, path, filename; demangle, target, kwargs...)
# Pick a compiler
cc = Sys.isapple() ? `cc` : clang()
if !isnothing(target.compiler)
cc = `$(target.compiler)`
else
cc = Sys.isapple() ? `cc` : clang()
end

# Compile!
if Sys.isapple()
# Apple no longer uses _start, so we can just specify a custom entry
Expand Down Expand Up @@ -318,8 +329,8 @@ end

"""
```julia
generate_shlib(f::Function, tt, [external::Bool=true], [path::String], [name], [filename]; kwargs...)
generate_shlib(funcs::Array, [external::Bool=true], [path::String], [filename::String]; demangle=true, kwargs...)
generate_shlib(f::Function, tt, [path::String], [name], [filename]; kwargs...)
generate_shlib(funcs::Array, [path::String], [filename::String]; demangle=true, target::StaticTarget=StaticTarget(), kwargs...)
```
Low level interface for compiling a shared object / dynamically loaded library
(`.so` / `.dylib`) for function `f` given a tuple type `tt` characterizing
Expand Down Expand Up @@ -356,75 +367,83 @@ julia> ccall(("test", "example/test.dylib"), Float64, (Int64,), 100_000)
5.2564961094956075
```
"""
function generate_shlib(f::Function, tt, external::Bool=true, path::String=tempname(), name=fix_name(f), filename=name; kwargs...)
generate_shlib(((f, tt),), external, path, filename; kwargs...)
function generate_shlib(f::Function, tt, path::String=tempname(), name=fix_name(f), filename=name; target=StaticTarget(), kwargs...)
generate_shlib(((f, tt),), path, filename; target, kwargs...)
end
# As above, but taking an array of functions and returning a single shlib
function generate_shlib(funcs::Union{Array,Tuple}, external::Bool=true, path::String=tempname(), filename::String="libfoo";
function generate_shlib(funcs::Union{Array,Tuple}, path::String=tempname(), filename::String="libfoo";
demangle = true,
cflags = ``,
target::StaticTarget=StaticTarget(),
kwargs...
)
if !isnothing(target.platform)
lib_path = joinpath(path, "$filename.$(platform_dlext(target.platform))")
else
lib_path = joinpath(path, "$filename.$(Libdl.dlext)")
end

lib_path = joinpath(path, "$filename.$(Libdl.dlext)")

_, obj_path = generate_obj(funcs, external, path, filename; demangle, kwargs...)
_, obj_path = generate_obj(funcs, path, filename; target, demangle, kwargs...)
# Pick a Clang
cc = Sys.isapple() ? `cc` : clang()
if !isnothing(target.compiler)
cc = `$(target.compiler)`
else
cc = Sys.isapple() ? `cc` : clang()
end
# Compile!
run(`$cc -shared $cflags $obj_path -o $lib_path `)

path, name
end

function native_code_llvm(@nospecialize(func), @nospecialize(types); kwargs...)
job, kwargs = native_job(func, types, true; kwargs...)
GPUCompiler.code_llvm(stdout, job; kwargs...)
function static_code_llvm(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...)
job, kwargs = static_job(func, types; target, kwargs...)
GPUCompiler.code_llvm(stdout, job; libraries=false, kwargs...)
end

function native_code_typed(@nospecialize(func), @nospecialize(types); kwargs...)
job, kwargs = native_job(func, types, true; kwargs...)
function static_code_typed(@nospecialize(func), @nospecialize(types); target::StaticTarget=StaticTarget(), kwargs...)
job, kwargs = static_job(func, types; target, kwargs...)
GPUCompiler.code_typed(job; kwargs...)
end

function native_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name(f); kwargs...)
job, kwargs = native_job(f, tt, true; fname, kwargs...)
GPUCompiler.code_native(stdout, job; kwargs...)
function static_code_native(@nospecialize(f), @nospecialize(tt), fname=fix_name(f); target::StaticTarget=StaticTarget(), kwargs...)
job, kwargs = static_job(f, tt; fname, target, kwargs...)
GPUCompiler.code_native(stdout, job; libraries=false, kwargs...)
end

# Return an LLVM module
function native_llvm_module(f, tt, name=fix_name(f); demangle, kwargs...)
function static_llvm_module(f, tt, name=fix_name(f); demangle=true, target::StaticTarget=StaticTarget(), kwargs...)
if !demangle
name = "julia_"*name
end
job, kwargs = native_job(f, tt, true; name, kwargs...)
job, kwargs = static_job(f, tt; name, target, kwargs...)
m = GPUCompiler.JuliaContext() do context
m, _ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false)
m, _ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false)
locate_pointers_and_runtime_calls(m)
m
end
return m
end

#Return an LLVM module for multiple functions
function native_llvm_module(funcs::Union{Array,Tuple}; demangle=true, kwargs...)
function static_llvm_module(funcs::Union{Array,Tuple}; demangle=true, target::StaticTarget=StaticTarget(), kwargs...)
f,tt = funcs[1]
mod = GPUCompiler.JuliaContext() do context
name_f = fix_name(f)
if !demangle
name_f = "julia_"*name_f
end
job, kwargs = native_job(f, tt, true; name = name_f, kwargs...)
mod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false)
job, kwargs = static_job(f, tt; name = name_f, target, kwargs...)
mod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false)
if length(funcs) > 1
for func in funcs[2:end]
f,tt = func
name_f = fix_name(f)
if !demangle
name_f = "julia_"*name_f
end
job, kwargs = native_job(f, tt, true; name = name_f, kwargs...)
tmod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false)
job, kwargs = static_job(f, tt; name = name_f, target, kwargs...)
tmod,_ = GPUCompiler.codegen(:llvm, job; strip=true, only_entry=false, validate=false, libraries=false)
link!(mod,tmod)
end
end
Expand Down Expand Up @@ -458,8 +477,8 @@ end

"""
```julia
generate_obj(f, tt, external::Bool, path::String = tempname(), filenamebase::String="obj";
target = (),
generate_obj(f, tt, path::String = tempname(), filenamebase::String="obj";
target::StaticTarget=StaticTarget(),
demangle = true,
strip_llvm = false,
strip_asm = true,
Expand All @@ -471,7 +490,7 @@ a tuple type `tt` characterizing the types of the arguments for which the
function will be compiled.

`target` can be used to change the output target. This is useful for compiling to WebAssembly and embedded targets.
This is a named tuple with fields `triple`, `cpu`, and `features` (each of these are strings).
This is a struct of the type StaticTarget()
The defaults compile to the native target.

If `demangle` is set to `false`, compiled function names are prepended with "julia_".
Expand All @@ -481,7 +500,7 @@ If `demangle` is set to `false`, compiled function names are prepended with "jul
julia> fib(n) = n <= 1 ? n : fib(n - 1) + fib(n - 2)
fib (generic function with 1 method)

julia> path, name, table = StaticCompiler.generate_obj_for_compile(fib, Tuple{Int64}, "./test")
julia> path, name, table = StaticCompiler.generate_obj(fib, Tuple{Int64}, "./test")
("./test", "fib", IdDict{Any, String}())

shell> tree \$path
Expand All @@ -498,8 +517,8 @@ end

"""
```julia
generate_obj(funcs::Union{Array,Tuple}, external::Bool, path::String = tempname(), filenamebase::String="obj";
target = (),
generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj";
target::StaticTarget=StaticTarget(),
demangle =false,
strip_llvm = false,
strip_asm = true,
Expand All @@ -511,21 +530,22 @@ Low level interface for compiling object code (`.o`) for an array of Tuples
which will be compiled.

`target` can be used to change the output target. This is useful for compiling to WebAssembly and embedded targets.
This is a named tuple with fields `triple`, `cpu`, and `features` (each of these are strings).
This is a struct of the type StaticTarget()
The defaults compile to the native target.
"""
function generate_obj(funcs::Union{Array,Tuple}, external::Bool, path::String = tempname(), filenamebase::String="obj";
function generate_obj(funcs::Union{Array,Tuple}, path::String = tempname(), filenamebase::String="obj";
demangle = true,
strip_llvm = false,
strip_asm = true,
opt_level = 3,
target::StaticTarget=StaticTarget(),
kwargs...)
f, tt = funcs[1]
mkpath(path)
obj_path = joinpath(path, "$filenamebase.o")
mod = native_llvm_module(funcs; demangle, kwargs...)
mod = static_llvm_module(funcs; demangle, kwargs...)
obj = GPUCompiler.JuliaContext() do ctx
fakejob, _ = native_job(f, tt, external; kwargs...)
fakejob, _ = static_job(f, tt; target, kwargs...)
obj, _ = GPUCompiler.emit_asm(fakejob, mod; strip=strip_asm, validate=false, format=LLVM.API.LLVMObjectFile)
obj
end
Expand Down
Loading
Loading