Skip to content

Commit 6fab6c3

Browse files
andrewrkandrewrk
authored
Merge pull request #12164 from ziglang/llvm-opt-byref
LLVM: lower optional types as byref=true
2 parents c26d9f6 + fe8c3ff commit 6fab6c3

File tree

1 file changed

+162
-66
lines changed

1 file changed

+162
-66
lines changed

‎src/codegen/llvm.zig

Lines changed: 162 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -4539,7 +4539,7 @@ pub const FuncGen = struct {
45394539
}
45404540
// We need to emit instructions to check for equality/inequality
45414541
// of optionals that are not pointers.
4542-
const is_by_ref = isByRef(operand_ty);
4542+
const is_by_ref = isByRef(scalar_ty);
45434543
const lhs_non_null = self.optIsNonNull(lhs, is_by_ref);
45444544
const rhs_non_null = self.optIsNonNull(rhs, is_by_ref);
45454545
const llvm_i2 = self.context.intType(2);
@@ -4564,8 +4564,8 @@ pub const FuncGen = struct {
45644564
_ = self.builder.buildBr(end_block);
45654565

45664566
self.builder.positionBuilderAtEnd(both_pl_block);
4567-
const lhs_payload = self.optPayloadHandle(lhs, is_by_ref);
4568-
const rhs_payload = self.optPayloadHandle(rhs, is_by_ref);
4567+
const lhs_payload = self.optPayloadHandle(lhs, scalar_ty);
4568+
const rhs_payload = self.optPayloadHandle(rhs, scalar_ty);
45694569
const payload_cmp = try self.cmp(lhs_payload, rhs_payload, payload_ty, op);
45704570
_ = self.builder.buildBr(end_block);
45714571
const both_pl_block_end = self.builder.getInsertBlock();
@@ -4740,12 +4740,14 @@ pub const FuncGen = struct {
47404740
const err_set_ty = try fg.dg.lowerType(Type.anyerror);
47414741
const zero = err_set_ty.constNull();
47424742
if (!payload_has_bits) {
4743+
// TODO add alignment to this load
47434744
const loaded = if (operand_is_ptr) fg.builder.buildLoad(err_union, "") else err_union;
47444745
break :err fg.builder.buildICmp(.NE, loaded, zero, "");
47454746
}
47464747
const err_field_index = errUnionErrorOffset(payload_ty, target);
47474748
if (operand_is_ptr or isByRef(err_union_ty)) {
47484749
const err_field_ptr = fg.builder.buildStructGEP(err_union, err_field_index, "");
4750+
// TODO add alignment to this load
47494751
const loaded = fg.builder.buildLoad(err_field_ptr, "");
47504752
break :err fg.builder.buildICmp(.NE, loaded, zero, "");
47514753
}
@@ -4765,13 +4767,22 @@ pub const FuncGen = struct {
47654767
if (!payload_has_bits) {
47664768
if (!operand_is_ptr) return null;
47674769

4768-
// TODO once we update to LLVM 14 this bitcast won't be necessary.
4770+
// TODO once we update to an LLVM version with opaque pointers
4771+
// this bitcast won't be necessary.
47694772
const res_ptr_ty = try fg.dg.lowerType(result_ty);
47704773
return fg.builder.buildBitCast(err_union, res_ptr_ty, "");
47714774
}
47724775
const offset = errUnionPayloadOffset(payload_ty, target);
47734776
if (operand_is_ptr or isByRef(payload_ty)) {
47744777
return fg.builder.buildStructGEP(err_union, offset, "");
4778+
} else if (isByRef(err_union_ty)) {
4779+
const payload_ptr = fg.builder.buildStructGEP(err_union, offset, "");
4780+
if (isByRef(payload_ty)) {
4781+
return payload_ptr;
4782+
}
4783+
const load_inst = fg.builder.buildLoad(payload_ptr, "");
4784+
load_inst.setAlignment(payload_ty.abiAlignment(target));
4785+
return load_inst;
47754786
}
47764787
return fg.builder.buildExtractValue(err_union, offset, "");
47774788
}
@@ -5730,12 +5741,7 @@ pub const FuncGen = struct {
57305741
// The payload and the optional are the same value.
57315742
return operand;
57325743
}
5733-
const index_type = self.context.intType(32);
5734-
const indices: [2]*const llvm.Value = .{
5735-
index_type.constNull(), // dereference the pointer
5736-
index_type.constNull(), // first field is the payload
5737-
};
5738-
return self.builder.buildInBoundsGEP(operand, &indices, indices.len, "");
5744+
return self.builder.buildStructGEP(operand, 0, "");
57395745
}
57405746

57415747
fn airOptionalPayloadPtrSet(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
@@ -5761,24 +5767,17 @@ pub const FuncGen = struct {
57615767
// Setting to non-null will be done when the payload is set.
57625768
return operand;
57635769
}
5764-
const index_type = self.context.intType(32);
5765-
{
5766-
// First set the non-null bit.
5767-
const indices: [2]*const llvm.Value = .{
5768-
index_type.constNull(), // dereference the pointer
5769-
index_type.constInt(1, .False), // second field is the non-null bit
5770-
};
5771-
const non_null_ptr = self.builder.buildInBoundsGEP(operand, &indices, indices.len, "");
5772-
_ = self.builder.buildStore(non_null_bit, non_null_ptr);
5773-
}
5770+
5771+
// First set the non-null bit.
5772+
const non_null_ptr = self.builder.buildStructGEP(operand, 1, "");
5773+
// TODO set alignment on this store
5774+
_ = self.builder.buildStore(non_null_bit, non_null_ptr);
5775+
57745776
// Then return the payload pointer (only if it's used).
57755777
if (self.liveness.isUnused(inst))
57765778
return null;
5777-
const indices: [2]*const llvm.Value = .{
5778-
index_type.constNull(), // dereference the pointer
5779-
index_type.constNull(), // first field is the payload
5780-
};
5781-
return self.builder.buildInBoundsGEP(operand, &indices, indices.len, "");
5779+
5780+
return self.builder.buildStructGEP(operand, 0, "");
57825781
}
57835782

57845783
fn airOptionalPayload(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
@@ -5795,7 +5794,7 @@ pub const FuncGen = struct {
57955794
return operand;
57965795
}
57975796

5798-
return self.optPayloadHandle(operand, isByRef(payload_ty));
5797+
return self.optPayloadHandle(operand, optional_ty);
57995798
}
58005799

58015800
fn airErrUnionPayload(
@@ -5807,6 +5806,8 @@ pub const FuncGen = struct {
58075806

58085807
const ty_op = self.air.instructions.items(.data)[inst].ty_op;
58095808
const operand = try self.resolveInst(ty_op.operand);
5809+
const operand_ty = self.air.typeOf(ty_op.operand);
5810+
const err_union_ty = if (operand_is_ptr) operand_ty.childType() else operand_ty;
58105811
const result_ty = self.air.typeOfIndex(inst);
58115812
const payload_ty = if (operand_is_ptr) result_ty.childType() else result_ty;
58125813
const target = self.dg.module.getTarget();
@@ -5821,6 +5822,14 @@ pub const FuncGen = struct {
58215822
const offset = errUnionPayloadOffset(payload_ty, target);
58225823
if (operand_is_ptr or isByRef(payload_ty)) {
58235824
return self.builder.buildStructGEP(operand, offset, "");
5825+
} else if (isByRef(err_union_ty)) {
5826+
const payload_ptr = self.builder.buildStructGEP(operand, offset, "");
5827+
if (isByRef(payload_ty)) {
5828+
return payload_ptr;
5829+
}
5830+
const load_inst = self.builder.buildLoad(payload_ptr, "");
5831+
load_inst.setAlignment(payload_ty.abiAlignment(target));
5832+
return load_inst;
58245833
}
58255834
return self.builder.buildExtractValue(operand, offset, "");
58265835
}
@@ -5874,16 +5883,11 @@ pub const FuncGen = struct {
58745883
_ = self.builder.buildStore(non_error_val, operand);
58755884
return operand;
58765885
}
5877-
const index_type = self.context.intType(32);
58785886
const target = self.dg.module.getTarget();
58795887
{
58805888
const error_offset = errUnionErrorOffset(payload_ty, target);
58815889
// First set the non-error value.
5882-
const indices: [2]*const llvm.Value = .{
5883-
index_type.constNull(), // dereference the pointer
5884-
index_type.constInt(error_offset, .False),
5885-
};
5886-
const non_null_ptr = self.builder.buildInBoundsGEP(operand, &indices, indices.len, "");
5890+
const non_null_ptr = self.builder.buildStructGEP(operand, error_offset, "");
58875891
const store_inst = self.builder.buildStore(non_error_val, non_null_ptr);
58885892
store_inst.setAlignment(Type.anyerror.abiAlignment(target));
58895893
}
@@ -5892,11 +5896,7 @@ pub const FuncGen = struct {
58925896
return null;
58935897

58945898
const payload_offset = errUnionPayloadOffset(payload_ty, target);
5895-
const indices: [2]*const llvm.Value = .{
5896-
index_type.constNull(), // dereference the pointer
5897-
index_type.constInt(payload_offset, .False),
5898-
};
5899-
return self.builder.buildInBoundsGEP(operand, &indices, indices.len, "");
5899+
return self.builder.buildStructGEP(operand, payload_offset, "");
59005900
}
59015901

59025902
fn airErrReturnTrace(self: *FuncGen, _: Air.Inst.Index) !?*const llvm.Value {
@@ -6391,8 +6391,30 @@ pub const FuncGen = struct {
63916391
const overflow_bit = self.builder.buildExtractValue(result_struct, 1, "");
63926392

63936393
var ty_buf: Type.Payload.Pointer = undefined;
6394-
const partial = self.builder.buildInsertValue(llvm_dest_ty.getUndef(), result, llvmFieldIndex(dest_ty, 0, tg, &ty_buf).?, "");
6395-
return self.builder.buildInsertValue(partial, overflow_bit, llvmFieldIndex(dest_ty, 1, tg, &ty_buf).?, "");
6394+
const result_index = llvmFieldIndex(dest_ty, 0, tg, &ty_buf).?;
6395+
const overflow_index = llvmFieldIndex(dest_ty, 1, tg, &ty_buf).?;
6396+
6397+
if (isByRef(dest_ty)) {
6398+
const target = self.dg.module.getTarget();
6399+
const alloca_inst = self.buildAlloca(llvm_dest_ty);
6400+
const result_alignment = dest_ty.abiAlignment(target);
6401+
alloca_inst.setAlignment(result_alignment);
6402+
{
6403+
const field_ptr = self.builder.buildStructGEP(alloca_inst, result_index, "");
6404+
const store_inst = self.builder.buildStore(result, field_ptr);
6405+
store_inst.setAlignment(result_alignment);
6406+
}
6407+
{
6408+
const field_ptr = self.builder.buildStructGEP(alloca_inst, overflow_index, "");
6409+
const store_inst = self.builder.buildStore(overflow_bit, field_ptr);
6410+
store_inst.setAlignment(1);
6411+
}
6412+
6413+
return alloca_inst;
6414+
}
6415+
6416+
const partial = self.builder.buildInsertValue(llvm_dest_ty.getUndef(), result, result_index, "");
6417+
return self.builder.buildInsertValue(partial, overflow_bit, overflow_index, "");
63966418
}
63976419

63986420
fn buildElementwiseCall(
@@ -6721,8 +6743,30 @@ pub const FuncGen = struct {
67216743
const overflow_bit = self.builder.buildICmp(.NE, lhs, reconstructed, "");
67226744

67236745
var ty_buf: Type.Payload.Pointer = undefined;
6724-
const partial = self.builder.buildInsertValue(llvm_dest_ty.getUndef(), result, llvmFieldIndex(dest_ty, 0, tg, &ty_buf).?, "");
6725-
return self.builder.buildInsertValue(partial, overflow_bit, llvmFieldIndex(dest_ty, 1, tg, &ty_buf).?, "");
6746+
const result_index = llvmFieldIndex(dest_ty, 0, tg, &ty_buf).?;
6747+
const overflow_index = llvmFieldIndex(dest_ty, 1, tg, &ty_buf).?;
6748+
6749+
if (isByRef(dest_ty)) {
6750+
const target = self.dg.module.getTarget();
6751+
const alloca_inst = self.buildAlloca(llvm_dest_ty);
6752+
const result_alignment = dest_ty.abiAlignment(target);
6753+
alloca_inst.setAlignment(result_alignment);
6754+
{
6755+
const field_ptr = self.builder.buildStructGEP(alloca_inst, result_index, "");
6756+
const store_inst = self.builder.buildStore(result, field_ptr);
6757+
store_inst.setAlignment(result_alignment);
6758+
}
6759+
{
6760+
const field_ptr = self.builder.buildStructGEP(alloca_inst, overflow_index, "");
6761+
const store_inst = self.builder.buildStore(overflow_bit, field_ptr);
6762+
store_inst.setAlignment(1);
6763+
}
6764+
6765+
return alloca_inst;
6766+
}
6767+
6768+
const partial = self.builder.buildInsertValue(llvm_dest_ty.getUndef(), result, result_index, "");
6769+
return self.builder.buildInsertValue(partial, overflow_bit, overflow_index, "");
67266770
}
67276771

67286772
fn airAnd(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
@@ -7347,11 +7391,9 @@ pub const FuncGen = struct {
73477391
}
73487392

73497393
comptime assert(optional_layout_version == 3);
7350-
const optional_llvm_ty = try self.dg.lowerType(optional_ty);
7394+
73517395
const non_null_bit = self.builder.buildNot(success_bit, "");
7352-
const non_null_field = self.builder.buildZExt(non_null_bit, self.dg.context.intType(8), "");
7353-
const partial = self.builder.buildInsertValue(optional_llvm_ty.getUndef(), payload, 0, "");
7354-
return self.builder.buildInsertValue(partial, non_null_field, 1, "");
7396+
return buildOptional(self, optional_ty, payload, non_null_bit);
73557397
}
73567398

73577399
fn airAtomicRmw(self: *FuncGen, inst: Air.Inst.Index) !?*const llvm.Value {
@@ -7515,11 +7557,13 @@ pub const FuncGen = struct {
75157557
const union_ptr = try self.resolveInst(bin_op.lhs);
75167558
const new_tag = try self.resolveInst(bin_op.rhs);
75177559
if (layout.payload_size == 0) {
7560+
// TODO alignment on this store
75187561
_ = self.builder.buildStore(new_tag, union_ptr);
75197562
return null;
75207563
}
75217564
const tag_index = @boolToInt(layout.tag_align < layout.payload_align);
75227565
const tag_field_ptr = self.builder.buildStructGEP(union_ptr, tag_index, "");
7566+
// TODO alignment on this store
75237567
_ = self.builder.buildStore(new_tag, tag_field_ptr);
75247568
return null;
75257569
}
@@ -8336,17 +8380,9 @@ pub const FuncGen = struct {
83368380
fn optIsNonNull(self: *FuncGen, opt_handle: *const llvm.Value, is_by_ref: bool) *const llvm.Value {
83378381
const field = b: {
83388382
if (is_by_ref) {
8339-
const index_type = self.context.intType(32);
8340-
8341-
const indices: [2]*const llvm.Value = .{
8342-
index_type.constNull(),
8343-
index_type.constInt(1, .False),
8344-
};
8345-
8346-
const field_ptr = self.builder.buildInBoundsGEP(opt_handle, &indices, indices.len, "");
8383+
const field_ptr = self.builder.buildStructGEP(opt_handle, 1, "");
83478384
break :b self.builder.buildLoad(field_ptr, "");
83488385
}
8349-
83508386
break :b self.builder.buildExtractValue(opt_handle, 1, "");
83518387
};
83528388
comptime assert(optional_layout_version == 3);
@@ -8355,18 +8391,63 @@ pub const FuncGen = struct {
83558391
}
83568392

83578393
/// Assumes the optional is not pointer-like and payload has bits.
8358-
fn optPayloadHandle(self: *FuncGen, opt_handle: *const llvm.Value, is_by_ref: bool) *const llvm.Value {
8359-
if (is_by_ref) {
8394+
fn optPayloadHandle(
8395+
fg: *FuncGen,
8396+
opt_handle: *const llvm.Value,
8397+
opt_ty: Type,
8398+
) *const llvm.Value {
8399+
var buf: Type.Payload.ElemType = undefined;
8400+
const payload_ty = opt_ty.optionalChild(&buf);
8401+
8402+
if (isByRef(opt_ty)) {
83608403
// We have a pointer and we need to return a pointer to the first field.
8361-
const index_type = self.context.intType(32);
8362-
const indices: [2]*const llvm.Value = .{
8363-
index_type.constNull(), // dereference the pointer
8364-
index_type.constNull(), // first field is the payload
8365-
};
8366-
return self.builder.buildInBoundsGEP(opt_handle, &indices, indices.len, "");
8404+
const payload_ptr = fg.builder.buildStructGEP(opt_handle, 0, "");
8405+
8406+
if (isByRef(payload_ty)) {
8407+
return payload_ptr;
8408+
}
8409+
const target = fg.dg.module.getTarget();
8410+
const payload_alignment = payload_ty.abiAlignment(target);
8411+
const load_inst = fg.builder.buildLoad(payload_ptr, "");
8412+
load_inst.setAlignment(payload_alignment);
8413+
return load_inst;
8414+
}
8415+
8416+
assert(!isByRef(payload_ty));
8417+
return fg.builder.buildExtractValue(opt_handle, 0, "");
8418+
}
8419+
8420+
fn buildOptional(
8421+
self: *FuncGen,
8422+
optional_ty: Type,
8423+
payload: *const llvm.Value,
8424+
non_null_bit: *const llvm.Value,
8425+
) !?*const llvm.Value {
8426+
const optional_llvm_ty = try self.dg.lowerType(optional_ty);
8427+
const non_null_field = self.builder.buildZExt(non_null_bit, self.dg.context.intType(8), "");
8428+
8429+
if (isByRef(optional_ty)) {
8430+
const target = self.dg.module.getTarget();
8431+
const alloca_inst = self.buildAlloca(optional_llvm_ty);
8432+
const payload_alignment = optional_ty.abiAlignment(target);
8433+
alloca_inst.setAlignment(payload_alignment);
8434+
8435+
{
8436+
const field_ptr = self.builder.buildStructGEP(alloca_inst, 0, "");
8437+
const store_inst = self.builder.buildStore(payload, field_ptr);
8438+
store_inst.setAlignment(payload_alignment);
8439+
}
8440+
{
8441+
const field_ptr = self.builder.buildStructGEP(alloca_inst, 1, "");
8442+
const store_inst = self.builder.buildStore(non_null_field, field_ptr);
8443+
store_inst.setAlignment(1);
8444+
}
8445+
8446+
return alloca_inst;
83678447
}
83688448

8369-
return self.builder.buildExtractValue(opt_handle, 0, "");
8449+
const partial = self.builder.buildInsertValue(optional_llvm_ty.getUndef(), payload, 0, "");
8450+
return self.builder.buildInsertValue(partial, non_null_field, 1, "");
83708451
}
83718452

83728453
fn fieldPtr(
@@ -9275,10 +9356,12 @@ fn ccAbiPromoteInt(
92759356
return null;
92769357
}
92779358

9359+
/// This is the one source of truth for whether a type is passed around as an LLVM pointer,
9360+
/// or as an LLVM value.
92789361
fn isByRef(ty: Type) bool {
92799362
// For tuples and structs, if there are more than this many non-void
92809363
// fields, then we make it byref, otherwise byval.
9281-
const max_fields_byval = 2;
9364+
const max_fields_byval = 0;
92829365

92839366
switch (ty.zigTypeTag()) {
92849367
.Type,
@@ -9332,10 +9415,23 @@ fn isByRef(ty: Type) bool {
93329415
return false;
93339416
},
93349417
.Union => return ty.hasRuntimeBits(),
9335-
.ErrorUnion => return isByRef(ty.errorUnionPayload()),
9418+
.ErrorUnion => {
9419+
const payload_ty = ty.errorUnionPayload();
9420+
if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
9421+
return false;
9422+
}
9423+
return true;
9424+
},
93369425
.Optional => {
93379426
var buf: Type.Payload.ElemType = undefined;
9338-
return isByRef(ty.optionalChild(&buf));
9427+
const payload_ty = ty.optionalChild(&buf);
9428+
if (!payload_ty.hasRuntimeBitsIgnoreComptime()) {
9429+
return false;
9430+
}
9431+
if (ty.optionalReprIsPayload()) {
9432+
return false;
9433+
}
9434+
return true;
93399435
},
93409436
}
93419437
}

0 commit comments

Comments
 (0)