fixes

base/atomics.jl

@@ -310,7 +310,7 @@ atomic_and!(x::Atomic, v) = (@atomic :acquire_release x.value & v).first
 atomic_or!(x::Atomic, v) = (@atomic :acquire_release x.value | v).first
 atomic_xor!(x::Atomic, v) = (@atomic :acquire_release x.value ⊻ v).first
 atomic_nand!(x::Atomic, v) = (@atomic :acquire_release x.value nand v).first
-atomic_xchg!(x::Atomic, v) = (@atomic :acquire_release x.value ((old,new)->new) v).first
+atomic_xchg!(x::Atomic, v) = (@atomicswap :acquire_release x.value = v)
 atomic_min!(x::Atomic, v) = (@atomic :acquire_release x.value min v).first
 atomic_max!(x::Atomic, v) = (@atomic :acquire_release x.value max v).first
 

src/aotcompile.cpp

@@ -487,7 +487,7 @@ static void resolve_workqueue(jl_codegen_params_t &params, egal_set &method_root
         }
         if (!preal_decl.empty()) {
             // merge and/or rename this prototype to the real function
-            if (Function *specfun = cast<Function>(mod->getNamedValue(preal_decl))) {
+            if (Function *specfun = cast_or_null<Function>(mod->getNamedValue(preal_decl))) {
                 if (proto.decl != specfun) {
                     proto.decl->replaceAllUsesWith(specfun);
                     if (proto.decl->hasFnAttribute(Attribute::InlineHint))
@@ -512,7 +512,7 @@ static void resolve_workqueue(jl_codegen_params_t &params, egal_set &method_root
             assert(ocinvokeDecl != "jl_fptr_const_return");
             assert(ocinvokeDecl != "jl_fptr_sparam");
             // merge and/or rename this prototype to the real function
-            if (Function *specfun = cast<Function>(mod->getNamedValue(ocinvokeDecl))) {
+            if (Function *specfun = cast_or_null<Function>(mod->getNamedValue(ocinvokeDecl))) {
                 if (proto.oc != specfun) {
                     proto.oc->replaceAllUsesWith(specfun);
                     if (proto.oc->hasFnAttribute(Attribute::InlineHint))

src/cgutils.cpp

@@ -2412,9 +2412,15 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
         instr = store;
     }
     else if (ismodifyfield && modifyop && !needlock && Order != AtomicOrdering::NotAtomic && !isboxed && realelty == elty && !intcast && elty->isIntegerTy() && !jl_type_hasptr(jltype)) {
-        // emit this only if we have a possiblity of optimizing it
+        // emit this only if we have a possibility of optimizing it
         if (Order == AtomicOrdering::Unordered)
             Order = AtomicOrdering::Monotonic;
+        if (jl_is_pointerfree(rhs.typ) && !rhs.isghost && (rhs.constant || rhs.isboxed || rhs.ispointer())) {
+            // if this value can be loaded from memory, do that now so that it is sequenced before the atomicmodify
+            // and the IR is less dependent on what was emitted before now to create this rhs.
+            // Inlining should do okay to clean this up later if there are parts we don't need.
+            rhs = jl_cgval_t(emit_unbox(ctx, julia_type_to_llvm(ctx, rhs.typ), rhs, rhs.typ), rhs.typ, NULL);
+        }
         bool gcstack_arg = JL_FEAT_TEST(ctx,gcstack_arg);
         Function *op = emit_modifyhelper(ctx, cmpop, *modifyop, jltype, elty, rhs, fname, gcstack_arg);
         std::string intr_name = "julia.atomicmodify.i";
@@ -2434,9 +2440,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx,
             Args.push_back(ctx.pgcstack);
         auto oldnew = ctx.builder.CreateCall(intr, Args);
         oldnew->addParamAttr(0, Attribute::getWithAlignment(oldnew->getContext(), Align(alignment)));
-        jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
-        ai.noalias = MDNode::concatenate(aliasscope, ai.noalias);
-        ai.decorateInst(oldnew);
+        //jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa);
+        //ai.noalias = MDNode::concatenate(aliasscope, ai.noalias);
+        //ai.decorateInst(oldnew);
         oldval = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 0), isboxed, jltype);
         rhs = mark_julia_type(ctx, ctx.builder.CreateExtractValue(oldnew, 1), isboxed, jltype);
     }

src/jitlayers.cpp

@@ -463,7 +463,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t
                 }
                 if (!preal_decl.empty()) {
                     // merge and/or rename this prototype to the real function
-                    if (Function *specfun = cast<Function>(mod->getNamedValue(preal_decl))) {
+                    if (Function *specfun = cast_or_null<Function>(mod->getNamedValue(preal_decl))) {
                         if (proto.decl != specfun) {
                             proto.decl->replaceAllUsesWith(specfun);
                             if (!proto.decl->isDeclaration() && specfun->isDeclaration())
@@ -505,7 +505,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t
                     assert(ocinvokeDecl != "jl_fptr_args");
                     assert(ocinvokeDecl != "jl_fptr_sparam");
                     // merge and/or rename this prototype to the real function
-                    if (Function *specfun = cast<Function>(mod->getNamedValue(ocinvokeDecl))) {
+                    if (Function *specfun = cast_or_null<Function>(mod->getNamedValue(ocinvokeDecl))) {
                         if (proto.oc != specfun) {
                             proto.oc->replaceAllUsesWith(specfun);
                             proto.oc->eraseFromParent();

src/llvm-expand-atomic-modify.cpp

@@ -112,6 +112,7 @@ std::pair<Value *, Value *> insertRMWCmpXchgLoop(
   std::prev(BB->end())->eraseFromParent();
   Builder.SetInsertPoint(BB);
   LoadInst *InitLoaded = Builder.CreateAlignedLoad(ResultTy, Addr, AddrAlign);
+  InitLoaded->setOrdering(AtomicOrdering::Unordered); // n.b. the original LLVM pass is missing this call so is actually mildly UB
   Builder.CreateBr(LoopBB);
 
   // Start the main loop block now that we've taken care of the preliminaries.