diff --git a/compiler/ci/ci_common/benchmark-builders.jsonnet b/compiler/ci/ci_common/benchmark-builders.jsonnet index 681e0bc62cdd..c8f109a0f9ab 100644 --- a/compiler/ci/ci_common/benchmark-builders.jsonnet +++ b/compiler/ci/ci_common/benchmark-builders.jsonnet @@ -81,6 +81,12 @@ for suite in bench.groups.main_suites ], + local shenandoah_builds = [ + c.weekly + hw.x52 + jdk + cc.libgraal + cc.shenandoah_mode + suite, + for jdk in cc.product_jdks + for suite in bench.groups.main_suites + [bench.specjbb2015] + ], + local metrics_suites = [bench.dacapo, bench.scala_dacapo, bench.renaissance, bench.specjvm2008], local metrics_builds = std.flattenArrays([ @@ -92,7 +98,8 @@ for suite in metrics_suites ]), - local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + zgc_builds + zgc_avx_builds + aarch64_builds + metrics_builds, + local all_builds = main_builds + weekly_amd64_forks_builds + weekly_aarch64_forks_builds + profiling_builds + avx_builds + zgc_builds + zgc_avx_builds + + shenandoah_builds + aarch64_builds + metrics_builds, local filtered_builds = [b for b in all_builds if b.is_jdk_supported(b.jdk_version) && b.is_arch_supported(b.arch)], // adds a "defined_in" field to all builds mentioning the location of this current file builds:: utils.add_defined_in(filtered_builds, std.thisFile), diff --git a/compiler/ci/ci_common/compiler-common.libsonnet b/compiler/ci/ci_common/compiler-common.libsonnet index 968ab6e99c37..c7c33fe3af5b 100644 --- a/compiler/ci/ci_common/compiler-common.libsonnet +++ b/compiler/ci/ci_common/compiler-common.libsonnet @@ -191,6 +191,13 @@ } }, + shenandoah_mode:: { + platform+:: "-shenandoah", + environment+: { + "JVM_CONFIG"+: "-shenandoah", + } + }, + serialgc_mode:: { platform+:: "-serialgc", environment+: { diff --git a/compiler/ci/ci_common/gate.jsonnet b/compiler/ci/ci_common/gate.jsonnet index 9800c3086d8c..972e88974ba3 100644 --- a/compiler/ci/ci_common/gate.jsonnet +++ b/compiler/ci/ci_common/gate.jsonnet @@ -101,9 +101,15 @@ test:: s.base(no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"), unittest_compiler:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"), unittest_truffle:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"), + test_zgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"), unittest_compiler_zgc:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"), unittest_truffle_zgc:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"), + + test_shenandoah:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"), + unittest_compiler_shenandoah:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"), + unittest_truffle_shenandoah:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"), + test_serialgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseSerialGC"), jacoco_gate_args:: ["--jacoco-omit-excluded", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov"], @@ -131,43 +137,25 @@ # Runs truffle tests in a mode similar to HotSpot's -Xcomp option # (i.e. compile immediately without background compilation). - truffle_xcomp:: s.base("build,unittest", + truffle_xcomp_base(extra_vm_args=""):: s.base("build,unittest", extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " + "-Dpolyglot.engine.CompileImmediately=true " + "-Dpolyglot.engine.BackgroundCompilation=false " + - "-Dtck.inlineVerifierInstrument=false", + "-Dtck.inlineVerifierInstrument=false" + extra_vm_args, extra_unittest_args="--verbose truffle") + { environment+: {"TRACE_COMPILATION": "true"}, logs+: ["*/*_compilation.log"], components+: ["truffle"], }, - truffle_xcomp_zgc:: s.base("build,unittest", - extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " + - "-Dpolyglot.engine.CompileImmediately=true " + - "-Dpolyglot.engine.BackgroundCompilation=false " + - "-Dtck.inlineVerifierInstrument=false " + - "-XX:+UseZGC", - extra_unittest_args="--verbose truffle") + { - environment+: {"TRACE_COMPILATION": "true"}, - logs+: ["*/*_compilation.log"], - components+: ["truffle"], - }, - - truffle_xcomp_serialgc:: s.base("build,unittest", - extra_vm_args="-Dpolyglot.engine.AllowExperimentalOptions=true " + - "-Dpolyglot.engine.CompileImmediately=true " + - "-Dpolyglot.engine.BackgroundCompilation=false " + - "-Dtck.inlineVerifierInstrument=false " + - "-XX:+UseSerialGC", - extra_unittest_args="--verbose truffle") + { - environment+: {"TRACE_COMPILATION": "true"}, - logs+: ["*/*_compilation.log"], - components+: ["truffle"], - }, + truffle_xcomp:: s.truffle_xcomp_base(), + truffle_xcomp_zgc:: s.truffle_xcomp_base(" -XX:+UseZGC"), + truffle_xcomp_shenandoah:: s.truffle_xcomp_base(" -XX:+UseShenandoahGC"), + truffle_xcomp_serialgc:: s.truffle_xcomp_base(" -XX:+UseSerialGC"), ctw:: s.base("build,ctw", no_warning_as_error=true), ctw_zgc:: s.base("build,ctw", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"), + ctw_shenandoah:: s.base("build,ctw", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"), ctw_economy:: s.base("build,ctweconomy", extra_vm_args="-Djdk.graal.CompilerConfiguration=economy"), ctw_phaseplan_fuzzing:: s.base("build,ctwphaseplanfuzzing"), @@ -175,11 +163,13 @@ # Runs some benchmarks as tests benchmarktest:: s.base("build,benchmarktest") + jmh_benchmark_test, benchmarktest_zgc:: s.base("build,benchmarktest", extra_vm_args="-XX:+UseZGC") + jmh_benchmark_test, + benchmarktest_shenandoah:: s.base("build,benchmarktest", extra_vm_args="-XX:+UseShenandoahGC") + jmh_benchmark_test, bootstrap:: s.base("build,bootstrap", no_warning_as_error=true), bootstrap_lite:: s.base("build,bootstraplite", no_warning_as_error=true), bootstrap_full:: s.base("build,bootstrapfullverify", no_warning_as_error=true), bootstrap_full_zgc:: s.base("build,bootstrapfullverify", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"), + bootstrap_full_shenandoah:: s.base("build,bootstrapfullverify", no_warning_as_error=true, extra_vm_args="-XX:+UseShenandoahGC"), bootstrap_economy:: s.base("build,bootstrapeconomy", no_warning_as_error=true, extra_vm_args="-Djdk.graal.CompilerConfiguration=economy"), style:: c.deps.eclipse + c.deps.jdt + c.deps.spotbugs + s.base("style,fullbuild,javadoc") + galahad.exclude, @@ -491,28 +481,45 @@ self.make_build(self.jdk_latest, "linux-amd64", "coverage_avx3").build ], - # Test ZGC on supported platforms. Windows requires version 1083 or later which will - # probably require adding some capabilities. - local all_zgc_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude - for jdk in [ - self.jdk_latest - ] - for os_arch in [ - "linux-amd64", - "linux-aarch64", - "darwin-amd64", - "darwin-aarch64" - ] - for task in [ - "test_zgc", - "unittest_compiler_zgc", - "unittest_truffle_zgc", - "truffle_xcomp_zgc", - "ctw_zgc", - "benchmarktest_zgc", - "bootstrap_full_zgc", - ] - ], + # Test ZGC on supported platforms. Windows requires version 1083 or later which will + # probably require adding some capabilities. + local all_zgc_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude + for jdk in [ + self.jdk_latest + ] + for os_arch in [ + "linux-amd64", + "linux-aarch64", + "darwin-amd64", + "darwin-aarch64" + ] + for task in [ + "test_zgc", + "unittest_compiler_zgc", + "unittest_truffle_zgc", + "truffle_xcomp_zgc", + "ctw_zgc", + "benchmarktest_zgc", + "bootstrap_full_zgc", + ] + ], + + # Test Shenandoah on supported platforms. + local all_shenandoah_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude + for jdk in [ + self.jdk_latest + ] + for os_arch in all_os_arches + for task in [ + "test_shenandoah", + "unittest_compiler_shenandoah", + "unittest_truffle_shenandoah", + "truffle_xcomp_shenandoah", + "ctw_shenandoah", + "benchmarktest_shenandoah", + "bootstrap_full_shenandoah", + ] + ], # Run unittests with SerialGC. local all_serialgc_builds = [self.make_build(self.jdk_latest, os_arch, task).build + galahad.exclude @@ -563,6 +570,7 @@ all_platforms_builds + all_coverage_builds + all_zgc_builds + + all_shenandoah_builds + all_serialgc_builds + style_builds + linux_amd64_jdk_latest_builds + diff --git a/compiler/ci/ci_includes/baseline-benchmarks.jsonnet b/compiler/ci/ci_includes/baseline-benchmarks.jsonnet index 3bb121270f9c..082799c47a48 100644 --- a/compiler/ci/ci_includes/baseline-benchmarks.jsonnet +++ b/compiler/ci/ci_includes/baseline-benchmarks.jsonnet @@ -60,6 +60,7 @@ local gc_variants_builds = std.flattenArrays([ [ c.monthly + hw.x52 + jdk + cc.c2 + cc.zgc_mode + suite, + c.monthly + hw.x52 + jdk + cc.c2 + cc.shenandoah_mode + suite, ] for jdk in cc.product_jdks for suite in bench.groups.main_suites @@ -68,6 +69,7 @@ c.monthly + hw.x52 + jdk + cc.c2 + cc.serialgc_mode + bench.microservice_benchmarks, c.monthly + hw.x52 + jdk + cc.c2 + cc.pargc_mode + bench.microservice_benchmarks, c.monthly + hw.x52 + jdk + cc.c2 + cc.zgc_mode + bench.microservice_benchmarks, + c.monthly + hw.x52 + jdk + cc.c2 + cc.shenandoah_mode + bench.microservice_benchmarks, ] for jdk in cc.product_jdks ]), diff --git a/compiler/mx.compiler/mx_graal_benchmark.py b/compiler/mx.compiler/mx_graal_benchmark.py index 67c5eeb735b3..2683e184ac89 100644 --- a/compiler/mx.compiler/mx_graal_benchmark.py +++ b/compiler/mx.compiler/mx_graal_benchmark.py @@ -48,6 +48,7 @@ ('zgc', ['-XX:+UseZGC'], 12), ('zgc-avx2', ['-XX:+UseZGC', '-XX:UseAVX=2'], 12), ('zgc-avx3', ['-XX:+UseZGC', '-XX:UseAVX=3'], 12), + ('shenandoah', ['-XX:+UseShenandoahGC'], 12), ('no-comp-oops', ['-XX:-UseCompressedOops'], 0), ('no-profile-info', ['-Djvmci.UseProfilingInformation=false'], 0), ('no-splitting', ['-Dpolyglot.engine.Splitting=false'], 0), diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/TransplantLowLevelGraphTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/TransplantLowLevelGraphTest.java index 2023a0d46353..22301acd9cbd 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/TransplantLowLevelGraphTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/core/test/TransplantLowLevelGraphTest.java @@ -95,7 +95,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec lateMacroInvoke.setTemplateProducer(new Supplier() { @Override public SnippetTemplate get() { - Arguments args = new Arguments(transplantTestSnippets.producer, GuardsStage.AFTER_FSA, LoweringTool.StandardLoweringStage.LOW_TIER); + Arguments args = new Arguments(transplantTestSnippets.producer, GuardsStage.AFTER_FSA, true, LoweringTool.StandardLoweringStage.LOW_TIER); // no args SnippetTemplate template = transplantTestSnippets.template(getProviders(), lateMacroInvoke, args); return template; @@ -118,7 +118,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec lateMacroInvoke.setTemplateProducer(new Supplier() { @Override public SnippetTemplate get() { - Arguments args = new Arguments(transplantTestSnippets.producerWithArgs, GuardsStage.AFTER_FSA, LoweringTool.StandardLoweringStage.LOW_TIER); + Arguments args = new Arguments(transplantTestSnippets.producerWithArgs, GuardsStage.AFTER_FSA, true, LoweringTool.StandardLoweringStage.LOW_TIER); args.add("a", arg0); args.add("b", arg1); SnippetTemplate template = transplantTestSnippets.template(getProviders(), lateMacroInvoke, args); @@ -142,7 +142,7 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec lateMacroInvoke.setTemplateProducer(new Supplier() { @Override public SnippetTemplate get() { - Arguments args = new Arguments(transplantTestSnippets.producerWithDeopt, GuardsStage.AFTER_FSA, LoweringTool.StandardLoweringStage.LOW_TIER); + Arguments args = new Arguments(transplantTestSnippets.producerWithDeopt, GuardsStage.AFTER_FSA, true, LoweringTool.StandardLoweringStage.LOW_TIER); args.add("a", arg0); args.add("b", arg1); SnippetTemplate template = transplantTestSnippets.template(getProviders(), lateMacroInvoke, args); diff --git a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/replacements/test/PointerTrackingTest.java b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/replacements/test/PointerTrackingTest.java index 547576256d48..cc6aa83d4059 100644 --- a/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/replacements/test/PointerTrackingTest.java +++ b/compiler/src/jdk.graal.compiler.test/src/jdk/graal/compiler/replacements/test/PointerTrackingTest.java @@ -24,23 +24,33 @@ */ package jdk.graal.compiler.replacements.test; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; + import jdk.graal.compiler.api.directives.GraalDirectives; +import jdk.graal.compiler.api.test.Graal; import jdk.graal.compiler.debug.DebugCloseable; import jdk.graal.compiler.debug.DebugContext; import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.HotSpotGraalRuntime; import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.graphbuilderconf.GraphBuilderContext; import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugin; import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins; import jdk.graal.compiler.nodes.graphbuilderconf.InvocationPlugins.Registration; import jdk.graal.compiler.replacements.Snippets; +import jdk.graal.compiler.runtime.RuntimeProvider; import jdk.graal.compiler.word.WordCastNode; -import org.junit.Test; - import jdk.vm.ci.meta.JavaKind; import jdk.vm.ci.meta.ResolvedJavaMethod; public class PointerTrackingTest extends ReplacementsTest implements Snippets { + @Before + public void before() { + HotSpotGraalRuntime runtime = (HotSpotGraalRuntime) Graal.getRequiredCapability(RuntimeProvider.class); + Assume.assumeTrue("doesn't aggressively move objects", runtime.getGarbageCollector() != HotSpotGraalRuntime.HotSpotGC.Shenandoah); + } @Test public void testTracking() { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java index ebde60a790ac..b980a9e7c265 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/amd64/AMD64NodeMatchRules.java @@ -390,7 +390,6 @@ public ComplexMatchResult testBitAndBranch(IfNode root, ValueNode value, Constan } @MatchRule("(If (IntegerTest Read=access value))") - @MatchRule("(If (IntegerTest FloatingRead=access value))") public ComplexMatchResult integerTestBranchMemory(IfNode root, LIRLowerableAccess access, ValueNode value) { return emitIntegerTestBranchMemory(root, value, access); } @@ -398,17 +397,10 @@ public ComplexMatchResult integerTestBranchMemory(IfNode root, LIRLowerableAcces @MatchRule("(If (IntegerEquals=compare value Read=access))") @MatchRule("(If (IntegerLessThan=compare value Read=access))") @MatchRule("(If (IntegerBelow=compare value Read=access))") - @MatchRule("(If (IntegerEquals=compare value FloatingRead=access))") - @MatchRule("(If (IntegerLessThan=compare value FloatingRead=access))") - @MatchRule("(If (IntegerBelow=compare value FloatingRead=access))") @MatchRule("(If (FloatEquals=compare value Read=access))") - @MatchRule("(If (FloatEquals=compare value FloatingRead=access))") @MatchRule("(If (FloatLessThan=compare value Read=access))") - @MatchRule("(If (FloatLessThan=compare value FloatingRead=access))") @MatchRule("(If (PointerEquals=compare value Read=access))") - @MatchRule("(If (PointerEquals=compare value FloatingRead=access))") @MatchRule("(If (ObjectEquals=compare value Read=access))") - @MatchRule("(If (ObjectEquals=compare value FloatingRead=access))") public ComplexMatchResult ifCompareMemory(IfNode root, CompareNode compare, ValueNode value, LIRLowerableAccess access) { return emitCompareBranchMemory(root, compare, value, access); } @@ -468,7 +460,6 @@ public ComplexMatchResult ifCompareLogicCas(IfNode root, CompareNode compare, Va return null; } - @MatchRule("(If (ObjectEquals=compare value FloatingRead=access))") public ComplexMatchResult ifLogicCas(IfNode root, CompareNode compare, ValueNode value, LIRLowerableAccess access) { return emitCompareBranchMemory(root, compare, value, access); } @@ -516,7 +507,6 @@ private ComplexMatchResult binaryRead(AMD64Assembler.VexRVMOp op, OperandSize si } @MatchRule("(Add value Read=access)") - @MatchRule("(Add value FloatingRead=access)") public ComplexMatchResult addMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -531,7 +521,6 @@ public ComplexMatchResult addMemory(ValueNode value, LIRLowerableAccess access) } @MatchRule("(Sub value Read=access)") - @MatchRule("(Sub value FloatingRead=access)") public ComplexMatchResult subMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -546,7 +535,6 @@ public ComplexMatchResult subMemory(ValueNode value, LIRLowerableAccess access) } @MatchRule("(Mul value Read=access)") - @MatchRule("(Mul value FloatingRead=access)") public ComplexMatchResult mulMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -561,7 +549,6 @@ public ComplexMatchResult mulMemory(ValueNode value, LIRLowerableAccess access) } @MatchRule("(And value Read=access)") - @MatchRule("(And value FloatingRead=access)") public ComplexMatchResult andMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -572,7 +559,6 @@ public ComplexMatchResult andMemory(ValueNode value, LIRLowerableAccess access) } @MatchRule("(Or value Read=access)") - @MatchRule("(Or value FloatingRead=access)") public ComplexMatchResult orMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -583,7 +569,6 @@ public ComplexMatchResult orMemory(ValueNode value, LIRLowerableAccess access) { } @MatchRule("(Xor value Read=access)") - @MatchRule("(Xor value FloatingRead=access)") public ComplexMatchResult xorMemory(ValueNode value, LIRLowerableAccess access) { OperandSize size = getMemorySize(access); if (size.isXmmType()) { @@ -660,20 +645,17 @@ public ComplexMatchResult writeNarrow(WriteNode root, NarrowNode narrow) { } @MatchRule("(SignExtend Read=access)") - @MatchRule("(SignExtend FloatingRead=access)") public ComplexMatchResult signExtend(SignExtendNode root, LIRLowerableAccess access) { return emitSignExtendMemory(access, root.getInputBits(), root.getResultBits(), null); } @MatchRule("(ZeroExtend Read=access)") - @MatchRule("(ZeroExtend FloatingRead=access)") public ComplexMatchResult zeroExtend(ZeroExtendNode root, LIRLowerableAccess access) { AMD64Kind memoryKind = getMemoryKind(access); return builder -> getArithmeticLIRGenerator().emitZeroExtendMemory(memoryKind, root.getResultBits(), (AMD64AddressValue) operand(access.getAddress()), getState(access)); } @MatchRule("(Narrow Read=access)") - @MatchRule("(Narrow FloatingRead=access)") public ComplexMatchResult narrowRead(NarrowNode root, LIRLowerableAccess access) { return new ComplexMatchResult() { @Override @@ -690,14 +672,12 @@ public Value evaluate(NodeLIRBuilder builder) { } @MatchRule("(SignExtend (Narrow=narrow Read=access))") - @MatchRule("(SignExtend (Narrow=narrow FloatingRead=access))") public ComplexMatchResult signExtendNarrowRead(SignExtendNode root, NarrowNode narrow, LIRLowerableAccess access) { LIRKind kind = getLIRGeneratorTool().getLIRKind(narrow.stamp(NodeView.DEFAULT)); return emitSignExtendMemory(access, narrow.getResultBits(), root.getResultBits(), kind); } @MatchRule("(FloatConvert Read=access)") - @MatchRule("(FloatConvert FloatingRead=access)") public ComplexMatchResult floatConvert(FloatConvertNode root, LIRLowerableAccess access) { if (root.getFloatConvert().getCategory().equals(FloatConvertCategory.FloatingPointToInteger) && (root.inputCanBeNaN() || root.canOverflow())) { /* We need to fix up the result of the conversion, the input should be in a register. */ @@ -730,7 +710,6 @@ public ComplexMatchResult floatConvert(FloatConvertNode root, LIRLowerableAccess } @MatchRule("(Reinterpret Read=access)") - @MatchRule("(Reinterpret FloatingRead=access)") public ComplexMatchResult reinterpret(ReinterpretNode root, LIRLowerableAccess access) { return builder -> { LIRKind kind = getLIRGeneratorTool().getLIRKind(root.stamp(NodeView.DEFAULT)); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java index 1215d3578f3c..1ec8d54f7445 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/common/GraalOptions.java @@ -193,6 +193,8 @@ public final class GraalOptions { // Register allocator debugging @Option(help = "Comma separated list of registers that register allocation is limited to.", type = OptionType.Debug) public static final OptionKey RegisterPressure = new OptionKey<>(null); + @Option(help = "Permit RegisterPressure setting to cause compilation to fail.", type = OptionType.Debug) + public static final OptionKey BailoutOnRegisterPressureFailure = new OptionKey<>(false); @Option(help = "Eliminates redundant conditional expressions and statements where possible. " + "This can improve performance because fewer logic instructions have to be executed.", type = OptionType.Expert) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/gen/LIRCompilerBackend.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/gen/LIRCompilerBackend.java index be4ffd4d71b8..f612f441116b 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/gen/LIRCompilerBackend.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/gen/LIRCompilerBackend.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.core.gen; +import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -114,12 +115,12 @@ public static LIRGenerationResult emitLIR(Backend backend, StructuredGraph graph try { return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo, entryPointDecorator); } catch (OutOfRegistersException e) { - if (allocationRestrictedTo != null) { + if (allocationRestrictedTo != null && !GraalOptions.BailoutOnRegisterPressureFailure.getValue(graph.getOptions())) { allocationRestrictedTo = null; return emitLIR0(backend, graph, stub, registerConfig, lirSuites, allocationRestrictedTo, entryPointDecorator); } /* If the re-execution fails we convert the exception into a "hard" failure */ - throw new GraalError(e); + throw new GraalError(e, "out of registers%s", allocationRestrictedTo == null ? "" : ": " + Arrays.toString(allocationRestrictedTo)); } finally { graph.checkCancellation(); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java index eae8407c56f9..ca71a2b3d400 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/EconomyLowTier.java @@ -26,6 +26,7 @@ import jdk.graal.compiler.debug.Assertions; import jdk.graal.compiler.graph.Graph; +import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.options.OptionValues; import jdk.graal.compiler.phases.PlaceholderPhase; import jdk.graal.compiler.phases.common.AddressLoweringPhase; @@ -37,6 +38,7 @@ import jdk.graal.compiler.phases.common.LowTierLoweringPhase; import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase; import jdk.graal.compiler.phases.common.TransplantGraphsPhase; +import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase; import jdk.graal.compiler.phases.schedule.SchedulePhase; import jdk.graal.compiler.phases.tiers.LowTierContext; @@ -51,6 +53,7 @@ public EconomyLowTier(OptionValues options) { appendPhase(new LowTierLoweringPhase(canonicalizer)); appendPhase(new ExpandLogicPhase(canonicalizer)); + appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION)); appendPhase(new EconomyPiRemovalPhase(canonicalizer)); if (Assertions.assertionsEnabled()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java index dd551716ec40..c3db9dd65c47 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/core/phases/LowTier.java @@ -28,6 +28,7 @@ import jdk.graal.compiler.core.common.GraalOptions; import jdk.graal.compiler.graph.Graph; +import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.options.Option; import jdk.graal.compiler.options.OptionKey; import jdk.graal.compiler.options.OptionType; @@ -47,6 +48,7 @@ import jdk.graal.compiler.phases.common.PropagateDeoptimizeProbabilityPhase; import jdk.graal.compiler.phases.common.RemoveOpaqueValuePhase; import jdk.graal.compiler.phases.common.TransplantGraphsPhase; +import jdk.graal.compiler.phases.common.WriteBarrierAdditionPhase; import jdk.graal.compiler.phases.schedule.SchedulePhase; import jdk.graal.compiler.phases.schedule.SchedulePhase.SchedulingStrategy; import jdk.graal.compiler.phases.tiers.LowTierContext; @@ -87,6 +89,8 @@ public LowTier(OptionValues options) { appendPhase(new FixReadsPhase(true, new SchedulePhase(GraalOptions.StressTestEarlyReads.getValue(options) ? SchedulingStrategy.EARLIEST : SchedulingStrategy.LATEST_OUT_OF_LOOPS_IMPLICIT_NULL_CHECKS))); + appendPhase(new WriteBarrierAdditionPhase(GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION)); + appendPhase(canonicalizerWithoutGVN); /* diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java index d5f7bc145040..f9744f7c00ca 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/graph/Node.java @@ -825,7 +825,7 @@ protected void updatePredecessor(Node oldSuccessor, Node newSuccessor) { } } - void initialize(Graph newGraph) { + final void initialize(Graph newGraph) { assertTrue(id == INITIAL_ID, "unexpected id: %d", id); this.graph = newGraph; newGraph.register(this); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java index a3d323714cd0..cc4fac762a43 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/GraalHotSpotVMConfig.java @@ -609,6 +609,44 @@ private long getZGCAddressField(String name) { public final long zBarrierSetRuntimeLoadBarrierOnOopArray = getZGCAddressField("ZBarrierSetRuntime::load_barrier_on_oop_array"); public final int zPointerLoadShift = getConstant("ZPointerLoadShift", Integer.class, -1, osArch.equals("aarch64")); + /* + * Shenandoah GC support. + */ + /** + * Indicates whether or not the HotSpot VM has been built with Shenandoah support. If not, then + * we don't expect the Shenandoah symbols to be present in JVMCI. + */ + public final boolean hasShenandoahGC = getStore().getConstants().containsKey("INCLUDE_SHENANDOAHGC") && getConstant("INCLUDE_SHENANDOAHGC", Boolean.class); + + /* + * Various Shenandoah GC constants. + */ + public final int shenandoahGCStateOffset = getConstant("ShenandoahThreadLocalData::gc_state_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahSATBIndexOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_index_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahSATBBufferOffset = getConstant("ShenandoahThreadLocalData::satb_mark_queue_buffer_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahCardTableOffset = getConstant("ShenandoahThreadLocalData::card_table_offset", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCRegionSizeBytesShift = getFieldValue("CompilerToVM::Data::shenandoah_region_size_bytes_shift", Integer.class, "int", -1, hasShenandoahGC); + public final long shenandoahGCCSetFastTestAddress = getFieldValue("CompilerToVM::Data::shenandoah_in_cset_fast_test_addr", Long.class, "address", -1L, hasShenandoahGC); + + public final int shenandoahGCStateHasForwarded = getConstant("ShenandoahHeap::HAS_FORWARDED", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateMarking = getConstant("ShenandoahHeap::MARKING", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateEvacuation = getConstant("ShenandoahHeap::EVACUATION", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateUpdateRefs = getConstant("ShenandoahHeap::UPDATE_REFS", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateWeakRoots = getConstant("ShenandoahHeap::WEAK_ROOTS", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateYoungMarking = getConstant("ShenandoahHeap::YOUNG_MARKING", Integer.class, -1, hasShenandoahGC); + public final int shenandoahGCStateOldMarking = getConstant("ShenandoahHeap::OLD_MARKING", Integer.class, -1, hasShenandoahGC); + + /* + * Shenandoah barrier slow-paths. + */ + public final long shenandoahLoadBarrierStrong = getAddress("ShenandoahRuntime::load_reference_barrier_strong", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierStrongNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_strong_narrow", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierWeak = getAddress("ShenandoahRuntime::load_reference_barrier_weak", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierWeakNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_weak_narrow", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierPhantom = getAddress("ShenandoahRuntime::load_reference_barrier_phantom", -1L, hasShenandoahGC); + public final long shenandoahLoadBarrierPhantomNarrow = getAddress("ShenandoahRuntime::load_reference_barrier_phantom_narrow", -1L, hasShenandoahGC); + public final long shenandoahWriteBarrierPre = getAddress("ShenandoahRuntime::write_barrier_pre", -1L, hasShenandoahGC); + // aarch64 specific nmethod entry barrier support // @formatter:off public final int BarrierSetAssembler_nmethod_patching_type = getFieldValue("CompilerToVM::Data::BarrierSetAssembler_nmethod_patching_type", Integer.class, "int", -1, osArch.equals("aarch64")); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java index 81d70f279c31..5895c1acfdd1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotBackendFactory.java @@ -296,6 +296,8 @@ private BarrierSet createBarrierSet(GraalHotSpotVMConfig config, MetaAccessProvi ResolvedJavaField referentField = HotSpotReplacementsUtil.referentField(metaAccess); if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new HotSpotZBarrierSet(objectArrayType, referentField); + } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new HotSpotShenandoahBarrierSet(objectArrayType, referentField, config); } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Epsilon) { return new NoBarrierSet(); } else if (config.useG1GC()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java index 45c845dd287f..46ddde53d680 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotGraalRuntime.java @@ -232,19 +232,23 @@ public enum HotSpotGC { Serial("UseSerialGC"), Parallel("UseParallelGC"), G1("UseG1GC"), - Z(true, true, flagIsSet("UseZGC")), - Epsilon(true, true, flagIsSet("UseEpsilonGC")), - - // Unsupported GCs - Shenandoah(false, true, flagIsSet("UseShenandoahGC")); + Epsilon("UseEpsilonGC"), + // GR-54355 ZGC and Shenandoah require extra work to support vectorization + Z("UseZGC", false), + Shenandoah("UseShenandoahGC", false); HotSpotGC(String flag) { - this(true, true, flagIsSet(flag)); + this(true, true, true, flagIsSet(flag)); + } + + HotSpotGC(String flag, boolean supportsVectorization) { + this(true, true, supportsVectorization, flagIsSet(flag)); } - HotSpotGC(boolean supported, boolean expectNamePresent, Predicate predicate) { + HotSpotGC(boolean supported, boolean expectNamePresent, boolean supportsVectorization, Predicate predicate) { this.supported = supported; this.expectNamePresent = expectNamePresent; + this.supportsVectorization = supportsVectorization; this.predicate = predicate; } @@ -258,6 +262,15 @@ private static Predicate flagIsSet(String flag) { */ final boolean supported; + /** + * Specifies if this GC supports vectorization of objects. + */ + final boolean supportsVectorization; + + public boolean supportsVectorization() { + return supportsVectorization; + } + /** * Specifies if {@link #name()} is expected to be present in the {@code CollectedHeap::Name} * C++ enum. diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java new file mode 100644 index 000000000000..ce2e6300a772 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/HotSpotShenandoahBarrierSet.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot; + +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.nodes.HotSpotCompressionNode; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahBarrierSet; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Specialization of {@link ShenandoahBarrierSet} that adds support for read barriers on handle + * locations and compressed references. + */ +public class HotSpotShenandoahBarrierSet extends ShenandoahBarrierSet { + private final CompressEncoding oopEncoding; + + public HotSpotShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField, GraalHotSpotVMConfig config) { + super(objectArrayType, referentField); + this.oopEncoding = config.getOopEncoding(); + this.useLoadRefBarrier = config.getFlag("ShenandoahLoadRefBarrier", Boolean.class); + this.useSATBBarrier = config.getFlag("ShenandoahSATBBarrier", Boolean.class); + this.useCASBarrier = config.getFlag("ShenandoahCASBarrier", Boolean.class); + this.useCardBarrier = config.getFlag("ShenandoahCardBarrier", Boolean.class); + } + + @Override + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.READ; + } + return super.barrierForLocation(currentBarrier, location, storageKind); + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + GraalError.guarantee(loadStamp instanceof AbstractObjectStamp, "expect object, got: %s", loadStamp); + return BarrierType.READ; + } + return super.readBarrierType(location, address, loadStamp); + } + + @Override + public BarrierType writeBarrierType(LocationIdentity location) { + if (location instanceof HotSpotReplacementsUtil.OopHandleLocationIdentity) { + return BarrierType.FIELD; + } + return BarrierType.NONE; + } + + @Override + protected ValueNode maybeUncompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + return HotSpotCompressionNode.uncompressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } + + @Override + protected ValueNode maybeCompressReference(ValueNode value, boolean narrow) { + if (value != null && narrow) { + return HotSpotCompressionNode.compressWithoutUnique(value.graph(), value, oopEncoding); + } + return value; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java index 68d2aaddba21..515d2683e5a1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotLIRGenerator.java @@ -63,6 +63,7 @@ import jdk.graal.compiler.hotspot.HotSpotLIRGenerator; import jdk.graal.compiler.hotspot.HotSpotLockStack; import jdk.graal.compiler.hotspot.aarch64.g1.AArch64HotSpotG1BarrierSetLIRTool; +import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.debug.BenchmarkCounters; import jdk.graal.compiler.hotspot.meta.HotSpotProviders; @@ -120,6 +121,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new AArch64HotSpotZBarrierSetLIRGenerator(config, providers); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new AArch64HotSpotShenandoahBarrierSetLIRGenerator(config, providers); + } return null; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java index a4f391c52c96..018ac4e93936 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/AArch64HotSpotMove.java @@ -112,6 +112,10 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { Register resultRegister = asRegister(result); Register ptr = asRegister(input); Register base = (isRegister(baseRegister) ? asRegister(baseRegister) : zr); + emitCompressCode(masm, ptr, resultRegister, base, encoding, nonNull); + } + + public static void emitCompressCode(AArch64MacroAssembler masm, Register ptr, Register resultRegister, Register base, CompressEncoding encoding, boolean nonNull) { // result = (ptr - base) >> shift if (!encoding.hasBase()) { if (encoding.hasShift()) { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java new file mode 100644 index 000000000000..1fba9c7d607e --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahBarrierSetLIRGenerator.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.core.aarch64.AArch64LIRGenerator; +import jdk.graal.compiler.core.aarch64.AArch64ReadBarrierSetLIRGenerator; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.memory.MemoryExtendKind; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRFrameState; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.PlatformKind; +import jdk.vm.ci.meta.Value; + +/** + * Lowers Shenandoah barriers to AArch64 LIR. + */ +public class AArch64HotSpotShenandoahBarrierSetLIRGenerator implements ShenandoahBarrierSetLIRGeneratorTool, AArch64ReadBarrierSetLIRGenerator { + public AArch64HotSpotShenandoahBarrierSetLIRGenerator(GraalHotSpotVMConfig config, HotSpotProviders providers) { + this.config = config; + this.providers = providers; + } + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + private static ForeignCallLinkage getReadBarrierStub(LIRGeneratorTool tool, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow) { + return switch (strength) { + case STRONG -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER); + case WEAK -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK); + case PHANTOM -> + narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM); + }; + } + + @Override + public Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull) { + PlatformKind platformKind = obj.getPlatformKind(); + LIRKind kind = LIRKind.reference(platformKind); + Value result = tool.newVariable(tool.toRegisterKind(kind)); + ForeignCallLinkage callTarget = getReadBarrierStub(tool, strength, narrow); + AllocatableValue object = tool.asAllocatable(obj); + AArch64AddressValue loadAddress = ((AArch64LIRGenerator) tool).asAddressValue(address, AArch64Address.ANY_SIZE); + tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + tool.append(new AArch64HotSpotShenandoahLoadRefBarrierOp(config, providers, tool.asAllocatable(result), object, loadAddress, callTarget, strength, notNull)); + return result; + } + + @Override + public void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull) { + AllocatableValue temp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + // If the assembly must load the value then it needs a temporary to store it. + AllocatableValue temp2 = expectedObject.equals(Value.ILLEGAL) ? lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)) : Value.ILLEGAL; + + // Load the address into a register + AllocatableValue addressValue = lirTool.newVariable(address.getValueKind()); + lirTool.emitMove(addressValue, address); + + ForeignCallLinkage callTarget = lirTool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_WRITE_BARRIER_PRE); + lirTool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + lirTool.append(new AArch64HotSpotShenandoahSATBBarrierOp(config, providers, addressValue, expectedObject, temp, temp2, callTarget, nonNull)); + } + + @Override + public void emitCardBarrier(LIRGeneratorTool lirTool, Value address) { + AArch64AddressValue addr = ((AArch64LIRGenerator) lirTool).asAddressValue(address, AArch64Address.ANY_SIZE); + AllocatableValue tmp = lirTool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + lirTool.append(new AArch64HotSpotShenandoahCardBarrierOp(config, providers, addr, tmp)); + } + + @Override + public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogic, Value address, MemoryOrderMode memoryOrder, AArch64Kind memKind, Variable result, + AllocatableValue allocatableExpectedValue, AllocatableValue allocatableNewValue, BarrierType barrierType) { + AllocatableValue tmp1 = tool.newVariable(LIRKind.value(AArch64Kind.QWORD)); + tool.append(new AArch64HotSpotShenandoahCompareAndSwapOp(config, providers, memKind, memoryOrder, isLogic, result, allocatableExpectedValue, allocatableNewValue, tool.asAllocatable(address), + tmp1)); + } + + @Override + public Value emitAtomicReadAndWrite(LIRGeneratorTool tool, LIRKind readKind, Value address, Value newValue, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.emitAtomicReadAndWrite(readKind, address, newValue, BarrierType.NONE); + } + + @Override + public Variable emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.getArithmetic().emitLoad(kind, address, state, memoryOrder, MemoryExtendKind.DEFAULT); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java new file mode 100644 index 000000000000..8e5cbced7a6a --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCardBarrierOp.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.aarch64.AArch64.zr; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah card barrier. + */ +public class AArch64HotSpotShenandoahCardBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahCardBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The store address. + */ + @Alive({COMPOSITE}) private AArch64AddressValue address; + + @Temp({REG}) private AllocatableValue tmp; + + protected AArch64HotSpotShenandoahCardBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AArch64AddressValue addr, AllocatableValue tmp) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = addr; + this.tmp = tmp; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L373-L392", + sha1 = "1c3e544b6fdec2f4ca0f07b2a1d5261d55754cb9") + // @formatter:on + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + try (AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister(); + AArch64MacroAssembler.ScratchRegister tmp3 = masm.getScratchRegister()) { + Register rtmp1 = asRegister(tmp); + Register rtmp2 = tmp2.getRegister(); + Register rtmp3 = tmp3.getRegister(); + AArch64Address storeAddr = address.toAddress(); + Register rthread = providers.getRegisters().getThreadRegister(); + + // Flatten address if necessary. + Register rAddr; + if (storeAddr.isBaseRegisterOnly()) { + rAddr = storeAddr.getBase(); + } else { + rAddr = rtmp1; + masm.loadAddress(rAddr, storeAddr); + } + + masm.lsr(64, rAddr, rAddr, HotSpotReplacementsUtil.cardTableShift(config)); + + AArch64Address currCTHolderAddr = AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED, rthread, + HotSpotReplacementsUtil.shenandoahCardTableOffset(config)); + masm.ldr(64, rtmp2, currCTHolderAddr); + + AArch64Address cardAddr = AArch64Address.createRegisterOffsetAddress(8, rAddr, rtmp2, false); + if (HotSpotReplacementsUtil.useCondCardMark(config)) { + Label alreadyDirty = new Label(); + masm.ldr(8, rtmp3, cardAddr); + masm.cbz(8, rtmp3, alreadyDirty); + masm.str(8, zr, cardAddr); + masm.bind(alreadyDirty); + } else { + masm.str(8, zr, cardAddr); + } + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java new file mode 100644 index 000000000000..8748fb665153 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahCompareAndSwapOp.java @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMove; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.aarch64.AArch64AtomicMove; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.aarch64.AArch64Kind; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.vm.ci.aarch64.AArch64.zr; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * Special Shenandoah CAS implementation that handles false negatives due to concurrent evacuation. + * The service is more complex than a traditional CAS operation because the CAS operation is + * intended to succeed if the reference at addr exactly matches expected or if the reference at addr + * holds a pointer to a from-space object that has been relocated to the location named by expected. + * There are two races that must be addressed: a) A parallel thread may mutate the contents of addr + * so that it points to a different object. In this case, the CAS operation should fail. b) A + * parallel thread may heal the contents of addr, replacing a from-space pointer held in addr with + * the to-space pointer representing the new location of the object. Upon entry to cmpxchg_oop, it + * is assured that new_val equals null or it refers to an object that is not being evacuated out of + * from-space, or it refers to the to-space version of an object that is being evacuated out of + * from-space. + */ +public class AArch64HotSpotShenandoahCompareAndSwapOp extends AArch64AtomicMove.CompareAndSwapOp { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahCompareAndSwapOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + @Temp private AllocatableValue tmp1Value; + + public AArch64HotSpotShenandoahCompareAndSwapOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AArch64Kind accessKind, MemoryOrderMode memoryOrder, boolean isLogicVariant, + Variable result, AllocatableValue expectedValue, AllocatableValue newValue, AllocatableValue address, AllocatableValue tmp1) { + super(TYPE, accessKind, memoryOrder, isLogicVariant, result, expectedValue, newValue, address); + this.providers = providers; + this.config = config; + + this.tmp1Value = tmp1; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L471-L606", + sha1 = "553a2fb0d37f39016eda85331e8cd2421153cbfe") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register address = asRegister(addressValue); + Register result = asRegister(resultValue); + Register expected = asRegister(expectedValue); + Register newVal = asRegister(newValue); + + Register tmp1 = asRegister(tmp1Value); + Label step2 = new Label(); + Label done = new Label(); + GraalError.guarantee(accessKind == AArch64Kind.QWORD || accessKind == AArch64Kind.DWORD, "must be 64 or 32 bit access"); + int size = (accessKind == AArch64Kind.QWORD) ? 64 : 32; + + // Step 1. Fast-path. + // + // Try to CAS with given arguments. If successful, then we are done. + + emitCompareAndSwap(masm, accessKind, address, result, expected, newVal, memoryOrder, true); + // EQ flag set iff success. result holds value fetched. + + // If expected equals null but result does not equal null, the + // step2 branches to done to report failure of CAS. If both + // expected and tmp2 equal null, the following branches to done to + // report success of CAS. There's no need for a special test of + // expected equal to null. + + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, step2); + + masm.bind(done); + + crb.getLIR().addSlowPath(this, () -> { + // Step 2. CAS has failed because the value held at addr does not + // match expected. This may be a false negative because the value fetched + // from addr (now held in result) may be a from-space pointer to the + // original copy of same object referenced by to-space pointer expected. + // + // To resolve this, it suffices to find the forward pointer associated + // with fetched value. If this matches expected, retry CAS with new + // parameters. If this mismatches, then we have a legitimate + // failure, and we're done. + masm.bind(step2); + + // Check for null. If we get null, then we have a legitimate failure. + masm.tst(size, result, result); + Label resultNullFailure = setConditionFlags ? new Label() : done; + masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, resultNullFailure); + + // overwrite tmp1 with from-space pointer fetched from memory + masm.mov(size, tmp1, result); + + // Decode tmp1 in order to resolve its forward pointer + uncompress(masm, tmp1); + + // Load mark-word (i.e. potential forwarding pointer). + masm.ldr(64, tmp1, AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED, tmp1, config.markOffset)); + // Invert the mark-word, so that we can test the two lowest bits for 11, while + // preserving the upper bits. + masm.eon(64, tmp1, tmp1, zr); + // Check lowest bits for 00, which would have been originally 11. + // Original 11 indicates a forwarded object. + masm.tst(64, tmp1, config.markWordLockMaskInPlace); + // If not forwarded, then we're done. It must be a legitimate failure. + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, done); + // Set lowest two bits, which will result in actual clearing the bits + // after the following inversion. + masm.orr(64, tmp1, tmp1, config.markWordLockMaskInPlace); + // ... and invert all bits back to get the forwarding pointer into tmp1. + masm.eon(64, tmp1, tmp1, zr); + + // Encode tmp1 to compare against expected. + compress(masm, tmp1); + + // Does forwarded value of fetched from-space pointer match original + // value of expected? + masm.cmp(size, tmp1, expected); + + // If not, then the failure was legitimate and we're done. + // Branching to done with NE condition denotes failure. + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, done); + + // Fall through to step 3. No need for step3 label. + + // Step 3. We've confirmed that the value originally held in memory + // (now held in result) pointed to from-space version of original + // expected value. Try the CAS again with the from-space expected + // value. If it now succeeds, we're good. + // + // Note: result holds encoded from-space pointer that matches to-space + // object residing at expected. result is the new "expected". + masm.mov(size, tmp1, result); + emitCompareAndSwap(masm, accessKind, address, result, tmp1, newVal, memoryOrder, true); + // EQ flag set iff success. result holds value fetched. + + // If fetched value did not equal the new expected, this could + // still be a false negative because some other thread may have + // newly overwritten the memory value with its to-space equivalent. + masm.branchConditionally(AArch64Assembler.ConditionFlag.EQ, done); + + // In the rare case that four steps are required to perform the + // requested operation, the fourth step is the same as the first. + + // Step 4. CAS has failed because the value most recently fetched + // from addr is no longer the from-space pointer held in result. If a + // different thread replaced the in-memory value with its equivalent + // to-space pointer, then CAS may still be able to succeed. The + // value held in the expected register has not changed. + // + // It is extremely rare we reach this point. + + emitCompareAndSwap(masm, accessKind, address, result, expected, newVal, memoryOrder, setConditionFlags); + // EQ flag set iff success. result holds value fetched. + + masm.jmp(done); + + if (setConditionFlags) { + masm.bind(resultNullFailure); + // Clear zero flag to indicate failure. We come here knowing that result is null, + // so comparing it to 1 results in the zero (EQ) flag getting cleared. + masm.subs(32, zr, result, 1); + masm.jmp(done); + } + }); + } + + void uncompress(AArch64MacroAssembler masm, Register obj) { + if (accessKind == AArch64Kind.DWORD) { + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AArch64HotSpotMove.UncompressPointer.emitUncompressCode(masm, obj, obj, heapBase, config.getOopEncoding(), false); + } + } + + void compress(AArch64MacroAssembler masm, Register obj) { + if (accessKind == AArch64Kind.DWORD) { + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AArch64HotSpotMove.CompressPointer.emitCompressCode(masm, obj, obj, heapBase, config.getOopEncoding(), false); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java new file mode 100644 index 000000000000..31689102b9eb --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahLoadRefBarrierOp.java @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64AddressValue; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; + +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah load-reference barrier. + */ +public class AArch64HotSpotShenandoahLoadRefBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahLoadRefBarrierOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + /** + * The slow-path entry for the load-reference-barrier. + */ + private final ForeignCallLinkage callTarget; + + /** + * Strength (strong, weak, phantom) of incoming object reference. This affects whether or not + * the barrier needs to be active in the weak-roots phase, and whether or not we need to check + * for the object to be in the collection set. + */ + private final ShenandoahLoadRefBarrierNode.ReferenceStrength strength; + + /** + * If we know that the incoming object is not null, then we don't need to emit a null-check. + */ + private final boolean notNull; + + /** + * The output of the LRB. Passes the canonicalized reference to the consumer. + */ + @Def({REG}) private AllocatableValue result; + + /** + * The input of the LRB. This is typically a reference that has just been loaded. + */ + @Use({REG}) private AllocatableValue object; + + @Alive({COMPOSITE}) private AArch64AddressValue loadAddress; + + public AArch64HotSpotShenandoahLoadRefBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue result, AllocatableValue object, AArch64AddressValue loadAddress, + ForeignCallLinkage callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength strength, + boolean notNull) { + super(TYPE); + this.providers = providers; + this.config = config; + this.result = result; + this.object = object; + this.loadAddress = loadAddress; + this.callTarget = callTarget; + this.strength = strength; + this.notNull = notNull; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register thread = providers.getRegisters().getThreadRegister(); + emitCode(config, crb, masm, this, thread, asRegister(result), asRegister(object), loadAddress.toAddress(), callTarget, strength, notNull); + } + + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L232-L309", + sha1 = "4ed44f985dfdca39bf93c6d306a378be4bf88fe7") + // @formatter:on + public static void emitCode(GraalHotSpotVMConfig config, CompilationResultBuilder crb, AArch64MacroAssembler masm, LIRInstruction op, Register thread, Register result, Register object, + AArch64Address loadAddress, ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean notNull) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register rscratch1 = sc1.getRegister(); + + Label done = new Label(); + Label csetCheck = new Label(); + Label slowPath = new Label(); + + // Move object to result, in case the heap is stable and no barrier needs to be called. + masm.mov(64, result, object); + + if (!notNull) { + // Check for object being null. + masm.cbz(64, result, done); + } + + // Check for heap stability + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, rscratch1, gcState); + if (strength != ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + // This is needed because in a short-cut cycle we may get a trailing + // weak-roots phase but no evacuation/update-refs phase, and during that, + // we need to take the LRB to report null for unreachable weak-refs. + // This is true even for non-cset objects. + // Two tests because HAS_FORWARDED | WEAK_ROOTS currently is not representable + // as a single immediate. + masm.tst(64, rscratch1, config.shenandoahGCStateHasForwarded); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slowPath); + masm.tst(64, rscratch1, config.shenandoahGCStateWeakRoots); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, slowPath); + } else { + masm.tst(64, rscratch1, config.shenandoahGCStateHasForwarded); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, csetCheck); + } + masm.bind(done); + + // Check for object in collection set in an out-of-line mid-path. + if (strength == ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + crb.getLIR().addSlowPath(op, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(csetCheck); + masm.mov(rtmp1, HotSpotReplacementsUtil.shenandoahGCCSetFastTestAddr(config)); + masm.lsr(64, rtmp2, object, HotSpotReplacementsUtil.shenandoahGCRegionSizeBytesShift(config)); + masm.ldr(8, rtmp2, AArch64Address.createRegisterOffsetAddress(8, rtmp1, rtmp2, false)); + masm.cbnz(32, rtmp2, slowPath); + masm.jmp(done); + } + }); + } + // Call runtime slow-path LRB in out-of-line slow-path. + crb.getLIR().addSlowPath(op, () -> { + try (AArch64MacroAssembler.ScratchRegister tmp1 = masm.getScratchRegister(); AArch64MacroAssembler.ScratchRegister tmp2 = masm.getScratchRegister()) { + Register rtmp1 = tmp1.getRegister(); + Register rtmp2 = tmp2.getRegister(); + masm.bind(slowPath); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 2, "Expecting callTarget to have only 2 parameters. It has %d", cc.getArgumentCount()); + + // Store first argument + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, object, cArg0); + + // Store second argument + Register addressReg; + if (loadAddress.isBaseRegisterOnly()) { + // Can directly use the base register as the address + addressReg = loadAddress.getBase(); + } else { + addressReg = rtmp1; + masm.loadAddress(addressReg, loadAddress); + } + AArch64Address cArg1 = (AArch64Address) crb.asAddress(cc.getArgument(1)); + masm.str(64, addressReg, cArg1); + + // Make the call + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : rtmp2, null); + + // Retrieve result and move to the result register. + AArch64Address cRet = (AArch64Address) crb.asAddress(cc.getReturn()); + masm.ldr(64, result, cRet); + masm.jmp(done); + } + }); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java new file mode 100644 index 000000000000..64f80b7a4c88 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/aarch64/shenandoah/AArch64HotSpotShenandoahSATBBarrierOp.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.aarch64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.aarch64.AArch64Address; +import jdk.graal.compiler.asm.aarch64.AArch64Assembler; +import jdk.graal.compiler.asm.aarch64.AArch64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMacroAssembler; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.aarch64.AArch64Call; +import jdk.graal.compiler.lir.aarch64.AArch64LIRInstruction; +import jdk.graal.compiler.lir.aarch64.AArch64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.asm.aarch64.AArch64Address.AddressingMode.IMMEDIATE_SIGNED_UNSCALED; +import static jdk.graal.compiler.core.common.GraalOptions.AssemblyGCBarriersSlowPathOnly; +import static jdk.graal.compiler.core.common.GraalOptions.VerifyAssemblyGCBarriers; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * AArch64 backend for the Shenandoah SATB barrier. + */ +public class AArch64HotSpotShenandoahSATBBarrierOp extends AArch64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AArch64HotSpotShenandoahSATBBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The SATB slow-path runtime entry. + */ + private final ForeignCallLinkage callTarget; + + /** + * If we know that the previous value is not null, then we don't need to emit a null-check. + */ + private final boolean nonNull; + + /** + * The store address. + */ + @Alive private Value address; + + /** + * The pre-loaded previous value, if any. + */ + @Alive({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value expectedObject; + + @Temp private Value temp; + + @Temp({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value temp2; + + public AArch64HotSpotShenandoahSATBBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue address, AllocatableValue expectedObject, AllocatableValue temp, AllocatableValue temp2, ForeignCallLinkage callTarget, boolean nonNull) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = address; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) ^ temp2.equals(Value.ILLEGAL), "only one register is necessary"); + this.expectedObject = expectedObject; + this.temp = temp; + this.temp2 = temp2; + this.callTarget = callTarget; + this.nonNull = nonNull; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) || expectedObject.getPlatformKind().getSizeInBytes() == 8, "expected uncompressed pointer"); + } + + public void loadObject(AArch64MacroAssembler masm, Register preVal, Register immediateAddress) { + if (config.useCompressedOops) { + masm.ldr(32, preVal, AArch64Address.createImmediateAddress(32, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + CompressEncoding encoding = config.getOopEncoding(); + AArch64Move.UncompressPointerOp.emitUncompressCode(masm, preVal, preVal, encoding, false, providers.getRegisters().getHeapBaseRegister(), false); + } else { + masm.ldr(64, preVal, AArch64Address.createImmediateAddress(64, IMMEDIATE_SIGNED_UNSCALED, immediateAddress, 0)); + } + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp#L100-L183", + sha1 = "7b3d183187ff6578e0d14eb54e4b5007ff4d5e1e") + // @formatter:on + protected void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { + Register storeAddress = asRegister(address); + Register thread = providers.getRegisters().getThreadRegister(); + Register tmp = asRegister(temp); + Register previousValue = expectedObject.equals(Value.ILLEGAL) ? asRegister(temp2) : asRegister(expectedObject); + + guaranteeDifferentRegisters(storeAddress, thread, tmp, previousValue); + + Label done = new Label(); + Label midPath = new Label(); + Label runtime = new Label(); + + // Is marking active? + int gcStateOffset = HotSpotReplacementsUtil.shenandoahGCStateOffset(config); + AArch64Address gcState = masm.makeAddress(8, thread, gcStateOffset); + masm.ldr(8, tmp, gcState); + masm.tst(64, tmp, config.shenandoahGCStateMarking); + masm.branchConditionally(AArch64Assembler.ConditionFlag.NE, midPath); + masm.bind(done); + + // Out of line mid-path. + crb.getLIR().addSlowPath(this, () -> { + masm.bind(midPath); + + // Do we need to load the previous value? + if (expectedObject.equals(Value.ILLEGAL)) { + loadObject(masm, previousValue, storeAddress); + } + + if (!nonNull) { + // Is the previous value null? + masm.cbz(64, previousValue, done); + } + + if (VerifyAssemblyGCBarriers.getValue(crb.getOptions())) { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register tmp2 = sc1.getRegister(); + verifyOop(masm, previousValue, tmp, tmp2, false); + } + } + + if (AssemblyGCBarriersSlowPathOnly.getValue(crb.getOptions())) { + masm.jmp(runtime); + } else { + int satbQueueIndexOffset = HotSpotReplacementsUtil.shenandoahSATBIndexOffset(config); + AArch64Address satbQueueIndex = masm.makeAddress(64, thread, satbQueueIndexOffset); + // tmp := *index_adr + // if tmp == 0 then goto runtime + masm.ldr(64, tmp, satbQueueIndex); + masm.cbz(64, tmp, runtime); + + // tmp := tmp - wordSize + // *index_adr := tmp + // tmp := tmp + *buffer_adr + masm.sub(64, tmp, tmp, 8); + masm.str(64, tmp, satbQueueIndex); + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + int satbQueueBufferOffset = HotSpotReplacementsUtil.shenandoahSATBBufferOffset(config); + AArch64Address satbQueueBuffer = masm.makeAddress(64, thread, satbQueueBufferOffset); + masm.ldr(64, scratch1, satbQueueBuffer); + masm.add(64, tmp, tmp, scratch1); + } + + // Record the previous value + masm.str(64, previousValue, masm.makeAddress(64, tmp, 0)); + masm.jmp(done); + } + }); + + // Out of line slow path + crb.getLIR().addSlowPath(this, () -> { + try (AArch64MacroAssembler.ScratchRegister sc1 = masm.getScratchRegister()) { + Register scratch1 = sc1.getRegister(); + masm.bind(runtime); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + AArch64Address cArg0 = (AArch64Address) crb.asAddress(cc.getArgument(0)); + masm.str(64, previousValue, cArg0); + AArch64Call.directCall(crb, masm, callTarget, AArch64Call.isNearCall(callTarget) ? null : scratch1, null); + masm.jmp(done); + } + }); + } + + private static void verifyOop(AArch64MacroAssembler masm, Register previousValue, Register tmp, Register tmp2, boolean compressed) { + ((AArch64HotSpotMacroAssembler) masm).verifyOop(previousValue, tmp, tmp2, compressed, true); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java index 59b9ab0ea770..1a4861b996ea 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64HotSpotLIRGenerator.java @@ -55,6 +55,7 @@ import jdk.graal.compiler.hotspot.HotSpotLIRGenerator; import jdk.graal.compiler.hotspot.HotSpotLockStack; import jdk.graal.compiler.hotspot.amd64.g1.AMD64HotSpotG1BarrierSetLIRTool; +import jdk.graal.compiler.hotspot.amd64.shenandoah.AMD64HotSpotShenandoahBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.amd64.z.AMD64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.debug.BenchmarkCounters; import jdk.graal.compiler.hotspot.meta.HotSpotForeignCallDescriptor; @@ -120,6 +121,9 @@ protected static BarrierSetLIRGeneratorTool getBarrierSet(GraalHotSpotVMConfig c if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { return new AMD64HotSpotZBarrierSetLIRGenerator(config, providers); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + return new AMD64HotSpotShenandoahBarrierSetLIRGenerator(config, providers); + } return null; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64X87MathSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64X87MathSnippets.java index c3baac840c89..1beb58903151 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64X87MathSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/AMD64X87MathSnippets.java @@ -107,7 +107,7 @@ public void lower(UnaryMathIntrinsicNode mathIntrinsicNode, LoweringTool tool) { throw GraalError.shouldNotReachHere("Snippet not found for math intrinsic " + mathIntrinsicNode.getOperation().name()); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(info, mathIntrinsicNode.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(info, mathIntrinsicNode.graph(), tool.getLoweringStage()); args.add("input", mathIntrinsicNode.getValue()); template(tool, mathIntrinsicNode, args).instantiate(tool.getMetaAccess(), mathIntrinsicNode, DEFAULT_REPLACER, tool, args); mathIntrinsicNode.safeDelete(); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java new file mode 100644 index 000000000000..083223b2126b --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahBarrierSetLIRGenerator.java @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.core.amd64.AMD64LIRGenerator; +import jdk.graal.compiler.core.amd64.AMD64ReadBarrierSetLIRGenerator; +import jdk.graal.compiler.core.common.LIRKind; +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.memory.MemoryExtendKind; +import jdk.graal.compiler.core.common.memory.MemoryOrderMode; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.LIRFrameState; +import jdk.graal.compiler.lir.Variable; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.amd64.AMD64Kind; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.PlatformKind; +import jdk.vm.ci.code.RegisterValue; +import jdk.vm.ci.meta.Value; + +public class AMD64HotSpotShenandoahBarrierSetLIRGenerator implements ShenandoahBarrierSetLIRGeneratorTool, AMD64ReadBarrierSetLIRGenerator { + public AMD64HotSpotShenandoahBarrierSetLIRGenerator(GraalHotSpotVMConfig config, HotSpotProviders providers) { + this.config = config; + this.providers = providers; + } + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + private static ForeignCallLinkage getReadBarrierStub(LIRGeneratorTool tool, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow) { + return switch (strength) { + case STRONG -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER); + case WEAK -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_WEAK); + case PHANTOM -> narrow ? tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW) + : tool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER_PHANTOM); + }; + } + + @Override + public Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull) { + PlatformKind platformKind = obj.getPlatformKind(); + LIRKind kind = LIRKind.reference(platformKind); + Value result = tool.newVariable(tool.toRegisterKind(kind)); + ForeignCallLinkage callTarget = getReadBarrierStub(tool, strength, narrow); + AllocatableValue object = tool.asAllocatable(obj); + AMD64AddressValue loadAddress = ((AMD64LIRGenerator) tool).asAddressValue(address); + AllocatableValue tmp = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + tool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + tool.append(new AMD64HotSpotShenandoahLoadRefBarrierOp(config, providers, tool.asAllocatable(result), object, loadAddress, callTarget, strength, tmp, tmp2, notNull)); + return result; + } + + @Override + public void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull) { + AllocatableValue temp = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + // If the assembly must load the value then it's needs a temporary to store it + AllocatableValue temp2 = expectedObject.equals(Value.ILLEGAL) ? lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)) : Value.ILLEGAL; + AllocatableValue temp3 = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + + // Load the address into a register + AllocatableValue addressValue = lirTool.newVariable(address.getValueKind()); + lirTool.emitMove(addressValue, address); + + ForeignCallLinkage callTarget = lirTool.getForeignCalls().lookupForeignCall(HotSpotHostForeignCallsProvider.SHENANDOAH_WRITE_BARRIER_PRE); + lirTool.getResult().getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + lirTool.append(new AMD64HotSpotShenandoahSATBBarrierOp(config, providers, addressValue, expectedObject, temp, temp2, temp3, callTarget, nonNull)); + } + + @Override + public void emitCardBarrier(LIRGeneratorTool lirTool, Value address) { + AMD64AddressValue addr = ((AMD64LIRGenerator) lirTool).asAddressValue(address); + AllocatableValue tmp = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = lirTool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + lirTool.append(new AMD64HotSpotShenandoahCardBarrierOp(config, providers, addr, tmp, tmp2)); + } + + @Override + public void emitCompareAndSwapOp(LIRGeneratorTool tool, boolean isLogic, LIRKind accessKind, AMD64Kind memKind, RegisterValue raxValue, AMD64AddressValue address, AllocatableValue newValue, + BarrierType barrierType) { + GraalError.guarantee(barrierType != BarrierType.NONE, "must have barrier type != NONE"); + AllocatableValue tmp1 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + AllocatableValue tmp2 = tool.newVariable(LIRKind.value(AMD64Kind.QWORD)); + tool.append(new AMD64HotSpotShenandoahCompareAndSwapOp(config, providers, memKind, raxValue, address, raxValue, newValue, tmp1, tmp2, isLogic)); + } + + @Override + public Value emitAtomicReadAndWrite(LIRGeneratorTool tool, LIRKind readKind, Value address, Value newValue, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.emitAtomicReadAndWrite(readKind, address, newValue, BarrierType.NONE); + } + + @Override + public Variable emitBarrieredLoad(LIRGeneratorTool tool, LIRKind kind, Value address, LIRFrameState state, MemoryOrderMode memoryOrder, BarrierType barrierType) { + // We insert the necessary barriers in the node graph, at that level it + // is easier to handle compressed object references. No need to do anything + // special here. + return tool.getArithmetic().emitLoad(kind, address, state, memoryOrder, MemoryExtendKind.DEFAULT); + } + +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java new file mode 100644 index 000000000000..25d483efcaf5 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCardBarrierOp.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.core.common.Stride; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah card barrier. + */ +public class AMD64HotSpotShenandoahCardBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahCardBarrierOp.class); + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The store address. + */ + @Alive({COMPOSITE}) private AMD64AddressValue address; + + @Temp({REG}) private AllocatableValue tmp; + + @Temp({REG}) private AllocatableValue tmp2; + + protected AMD64HotSpotShenandoahCardBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AMD64AddressValue addr, AllocatableValue tmp, AllocatableValue tmp2) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = addr; + this.tmp = tmp; + this.tmp2 = tmp2; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L509-L535", + sha1 = "ad163e79b0707221700bb3b2230581fb711ded61") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register rtmp1 = asRegister(tmp); + Register rtmp2 = asRegister(tmp2); + Register rthread = providers.getRegisters().getThreadRegister(); + guaranteeDifferentRegisters(rtmp1, rtmp2, rthread); + + masm.leaq(rtmp1, address.toAddress(masm)); + masm.shrq(rtmp1, HotSpotReplacementsUtil.cardTableShift(config)); + + AMD64Address currCTHolderAddr = new AMD64Address(rthread, HotSpotReplacementsUtil.shenandoahCardTableOffset(config)); + masm.movq(rtmp2, currCTHolderAddr); + + AMD64Address cardAddr = new AMD64Address(rtmp1, rtmp2, Stride.S1); + if (HotSpotReplacementsUtil.useCondCardMark(config)) { + Label alreadyDirty = new Label(); + masm.cmpb(cardAddr, 0 /* dirtyCardValue */); + masm.jccb(AMD64Assembler.ConditionFlag.Equal, alreadyDirty); + masm.movb(cardAddr, 0 /* dirtyCardValue */); + masm.bind(alreadyDirty); + } else { + masm.movb(cardAddr, 0 /* dirtyCardValue */); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java new file mode 100644 index 000000000000..ddf8321343f6 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahCompareAndSwapOp.java @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.Opcode; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.amd64.AMD64; +import jdk.vm.ci.amd64.AMD64Kind; + +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.vm.ci.code.ValueUtil.asRegister; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +/** + * Special Shenandoah CAS implementation that handles false negatives due to concurrent evacuation. + * The service is more complex than a traditional CAS operation because the CAS operation is + * intended to succeed if the reference at addr exactly matches expected or if the reference at addr + * holds a pointer to a from-space object that has been relocated to the location named by expected. + * There are two races that must be addressed: a) A parallel thread may mutate the contents of addr + * so that it points to a different object. In this case, the CAS operation should fail. b) A + * parallel thread may heal the contents of addr, replacing a from-space pointer held in addr with + * the to-space pointer representing the new location of the object. Upon entry to cmpxchg_oop, it + * is assured that new_val equals null or it refers to an object that is not being evacuated out of + * from-space, or it refers to the to-space version of an object that is being evacuated out of + * from-space. + */ +@Opcode("CAS_Shenandoah") +public class AMD64HotSpotShenandoahCompareAndSwapOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahCompareAndSwapOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + private final AMD64Kind accessKind; + private final boolean isLogic; + + @Def private AllocatableValue result; + @Alive({COMPOSITE}) private AMD64AddressValue address; + @Alive private AllocatableValue cmpValue; + @Alive private AllocatableValue newValue; + + @Temp private AllocatableValue tmp1Value; + @Temp private AllocatableValue tmp2Value; + + public AMD64HotSpotShenandoahCompareAndSwapOp(GraalHotSpotVMConfig config, HotSpotProviders providers, AMD64Kind accessKind, AllocatableValue result, AMD64AddressValue address, + AllocatableValue cmpValue, AllocatableValue newValue, AllocatableValue tmp1, AllocatableValue tmp2, boolean isLogic) { + super(TYPE); + this.providers = providers; + this.config = config; + this.accessKind = accessKind; + this.isLogic = isLogic; + this.result = result; + this.address = address; + this.cmpValue = cmpValue; + this.newValue = newValue; + this.tmp1Value = tmp1; + this.tmp2Value = tmp2; + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L596-L737", + sha1 = "c236c9ca8ccf7e45757d6ac04d16a230df9475a0") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + AMD64Address addr = address.toAddress(masm); + Register resultReg = asRegister(result); + Register expectedReg = asRegister(cmpValue); + Register newVal = asRegister(newValue); + + Register tmp1 = asRegister(tmp1Value); + Register tmp2 = asRegister(tmp2Value); + + GraalError.guarantee(resultReg.equals(expectedReg), "same registers"); + GraalError.guarantee(!tmp1.equals(tmp2), "different registers"); + GraalError.guarantee(!tmp1.equals(newVal), "different registers"); + GraalError.guarantee(!tmp1.equals(expectedReg), "different registers"); + GraalError.guarantee(!tmp2.equals(newVal), "different registers"); + GraalError.guarantee(!tmp2.equals(expectedReg), "different registers"); + GraalError.guarantee(!newVal.equals(expectedReg), "different registers"); + + Label done = new Label(); + Label step2 = new Label(); + + AMD64BaseAssembler.OperandSize opSize = AMD64BaseAssembler.OperandSize.get(accessKind); + + // Remember expectedReg for retry logic below + mov(masm, opSize, tmp1, expectedReg); + + // Step 1. Fast-path. + // + // Try to CAS with given arguments. If successful, then we are done. + + // There are two ways to reach this label. Initial entry into the + // cmpxchg_oop code expansion starts at step1 (which is equivalent + // to label step4). Additionally, in the rare case that four steps + // are required to perform the requested operation, the fourth step + // is the same as the first. On a second pass through step 1, + // control may flow through step 2 on its way to failure. It will + // not flow from step 2 to step 3 since we are assured that the + // memory at addr no longer holds a from-space pointer. + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + masm.jcc(AMD64Assembler.ConditionFlag.NotEqual, step2); + masm.bind(done); + + crb.getLIR().addSlowPath(this, () -> { + // Step 2. CAS had failed. This may be a false negative. + // + // The trouble comes when we compare the to-space pointer with the from-space + // pointer to the same object. To resolve this, it will suffice to resolve + // the value from memory -- this will give both to-space pointers. + // If they mismatch, then it was a legitimate failure. + masm.bind(step2); + + // First we check the value that we just fetched for null. + // If the value is null, then we have a legitimate failure. + // We need the correct ZF at the done label. That is, upon seeing null, + // we need to have ZF=0 to indicate CAS failure. Unfortunately, the + // null-test comes out with ZF=1. Therefore we branch to a small block + // at the end, which clears ZF, and then jumps to done. + Label resultNullFailure = isLogic ? new Label() : done; + masm.testAndJcc(opSize, resultReg, resultReg, AMD64Assembler.ConditionFlag.Zero, resultNullFailure, isLogic); + + // We need to preserve resultReg for the failure paths. + mov(masm, opSize, tmp2, resultReg); + // Uncompress the offending value that we just fetched, + // so that we can read its mark-word. + uncompress(masm, opSize, tmp2); + + // Resolve forwarding ptr into tmp2. + masm.movq(tmp2, new AMD64Address(tmp2, config.markOffset)); + // Invert the whole mark-word, to preserve the upper bits. + // If the object has been forwarded, i.e. the lowest + // two bits have been 11, then these become 00, which is + // easily testable. + masm.notq(tmp2); + // Test lowest two bits for 00. + GraalError.guarantee(config.markWordLockMaskInPlace == 3, "mask must be 3 (11)"); + // If not forwarded, then we have a legitimate failure of the CAS. + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, tmp2, (int) config.markWordLockMaskInPlace, AMD64Assembler.ConditionFlag.NotZero, done, false); + // Now set the two lowest bits. Upon re-inversion, these become 00. + masm.orq(tmp2, (int) config.markWordLockMaskInPlace); + // Invert again to get the resolved forwardee in tmp2. + masm.notq(tmp2); + // We need to compress it for comparing with original + // expectedReg value. + compress(masm, opSize, tmp2); + + // Now we have the forwarded offender in tmp2. + // Compare with original expectedReg value, and if they don't match, we have legitimate + // failure + masm.cmpAndJcc(opSize, tmp1, tmp2, AMD64Assembler.ConditionFlag.NotEqual, done, false); + + // Step 3. We've confirmed that the value originally held in memory + // (now held in resultReg/expectedReg) pointed to from-space version of original + // expectedReg value. Try the CAS again with the from-space expectedReg + // value. If it now succeeds, we're good. + // + // Note: resultReg holds encoded from-space pointer that matches to-space + // object residing at expectedReg. resultReg is the new "expectedReg". + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + + // If fetched value did not equal the new expectedReg, this could + // still be a false negative because some other thread may have + // newly overwritten the memory value with its to-space equivalent. + masm.jcc(AMD64Assembler.ConditionFlag.Equal, done); + + // Step 4. CAS has failed because the value most recently fetched + // from addr is no longer the from-space pointer held in tmp2. If a + // different thread replaced the in-memory value with its equivalent + // to-space pointer, then CAS may still be able to succeed. The + // value held in the expectedReg register must be updated to expect + // the original expectedReg value (cmpxchg uses the same register for + // expectedReg and resultReg, and it might hold a wrong value now). + mov(masm, opSize, expectedReg, tmp1); + emitCompareAndSwap(crb, masm, accessKind, addr, expectedReg, newVal, resultReg); + masm.jmp(done); + + if (isLogic) { + masm.bind(resultNullFailure); + // Clear 0 register to indicate failure. We arrive here with + // resultReg == 0, but need the ZR flag to be cleared to + // indicate failure on the done label. + masm.cmpl(resultReg, 1); + masm.jmp(done); + } + }); + } + + private static void mov(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst, Register src) { + if (opSize == AMD64BaseAssembler.OperandSize.QWORD) { + masm.movq(dst, src); + } else if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + masm.movl(dst, src); + } else { + GraalError.shouldNotReachHereUnexpectedValue(opSize); + } + } + + private void compress(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst) { + if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + CompressEncoding encoding = config.getOopEncoding(); + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AMD64Move.CompressPointerOp.emitCompressCode(masm, dst, encoding.getShift(), heapBase, true); + } else { + GraalError.guarantee(opSize == AMD64BaseAssembler.OperandSize.QWORD, "unexpected opSize"); + } + } + + private void uncompress(AMD64MacroAssembler masm, AMD64BaseAssembler.OperandSize opSize, Register dst) { + if (opSize == AMD64BaseAssembler.OperandSize.DWORD) { + CompressEncoding encoding = config.getOopEncoding(); + Register heapBase = providers.getRegisters().getHeapBaseRegister(); + AMD64Move.UncompressPointerOp.emitUncompressCode(masm, dst, encoding.getShift(), heapBase, true); + } else { + GraalError.guarantee(opSize == AMD64BaseAssembler.OperandSize.QWORD, "unexpected opSize"); + } + } + + private static void emitCompareAndSwap(CompilationResultBuilder crb, AMD64MacroAssembler masm, AMD64Kind accessKind, AMD64Address address, Register expected, Register newValue, Register result) { + GraalError.guarantee(expected.equals(AMD64.rax), "expected must be in rax"); + GraalError.guarantee(result.equals(AMD64.rax), "result must be in rax"); + + if (crb.target.isMP) { + masm.lock(); + } + switch (accessKind) { + case DWORD: + masm.cmpxchgl(address, newValue); + break; + case QWORD: + masm.cmpxchgq(newValue, address); + break; + default: + throw GraalError.shouldNotReachHereUnexpectedValue(accessKind); // ExcludeFromJacocoGeneratedReport + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java new file mode 100644 index 000000000000..62f2bd580a0b --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahLoadRefBarrierOp.java @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstruction; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64AddressValue; +import jdk.graal.compiler.lir.amd64.AMD64Call; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; + +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.COMPOSITE; +import static jdk.graal.compiler.lir.LIRInstruction.OperandFlag.REG; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah load-reference barrier. + */ +public class AMD64HotSpotShenandoahLoadRefBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahLoadRefBarrierOp.class); + + private final HotSpotProviders providers; + private final GraalHotSpotVMConfig config; + + /** + * The slow-path entry for the load-reference-barrier. + */ + private final ForeignCallLinkage callTarget; + + /** + * Strength (strong, weak, phantom) of incoming object reference. This affects whether or not + * the barrier needs to be active in the weak-roots phase, and whether or not we need to check + * for the object to be in the collection set. + */ + private final ShenandoahLoadRefBarrierNode.ReferenceStrength strength; + + /** + * If we know that the incoming object is not null, then we don't need to emit a null-check. + */ + private final boolean notNull; + + @Temp({REG}) private AllocatableValue tmp; + + @Temp({REG}) private AllocatableValue tmp2; + + /** + * The output of the LRB. Passes the canonicalized reference to the consumer. + */ + @Def({REG}) private AllocatableValue result; + + /** + * The input of the LRB. This is typically a reference that has just been loaded. + */ + @Alive({REG}) private AllocatableValue object; + + /** + * The address from where the reference has been loaded, if any. + */ + @Alive({COMPOSITE}) private AMD64AddressValue loadAddress; + + public AMD64HotSpotShenandoahLoadRefBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue result, AllocatableValue object, AMD64AddressValue loadAddress, + ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, + AllocatableValue tmp, AllocatableValue tmp2, boolean notNull) { + super(TYPE); + this.providers = providers; + this.config = config; + this.result = result; + this.object = object; + this.loadAddress = loadAddress; + this.callTarget = callTarget; + this.strength = strength; + this.notNull = notNull; + this.tmp = tmp; + this.tmp2 = tmp2; + } + + @Override + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register thread = providers.getRegisters().getThreadRegister(); + Register rtmp1 = asRegister(tmp); + Register rtmp2 = asRegister(tmp2); + Register objectRegister = asRegister(object); + Register resultRegister = asRegister(result); + AMD64Address loadAddr = loadAddress.toAddress(masm); + emitCode(config, crb, masm, this, thread, resultRegister, objectRegister, rtmp1, rtmp2, loadAddr, callTarget, strength, notNull); + } + + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L296-L430", + sha1 = "a039ddb87ee03446a7d015f2d955eb3014c9413e") + // @formatter:on + public static void emitCode(GraalHotSpotVMConfig config, CompilationResultBuilder crb, AMD64MacroAssembler masm, LIRInstruction op, Register thread, Register resultRegister, + Register objectRegister, Register rtmp1, Register rtmp2, + AMD64Address loadAddr, ForeignCallLinkage callTarget, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean notNull) { + guaranteeDifferentRegisters(thread, rtmp1, rtmp2, objectRegister, resultRegister); + + Label done = new Label(); + Label csetCheck = new Label(); + Label slowPath = new Label(); + + // Move object to result, in case the heap is stable and no barrier needs to be called. + masm.movq(resultRegister, objectRegister); + + if (!notNull) { + // Check for object being null. + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, resultRegister, resultRegister, AMD64Assembler.ConditionFlag.Zero, done, true); + } + + // Check for heap stability + masm.movb(rtmp1, new AMD64Address(thread, HotSpotReplacementsUtil.shenandoahGCStateOffset(config))); + if (strength != ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + // This is needed because in a short-cut cycle we may get a trailing + // weak-roots phase but no evacuation/update-refs phase, and during that, + // we need to take the LRB to report null for unreachable weak-refs. + // This is true even for non-cset objects. + masm.testlAndJcc(rtmp1, config.shenandoahGCStateHasForwarded | config.shenandoahGCStateWeakRoots, AMD64Assembler.ConditionFlag.NotZero, slowPath, false); + } else { + masm.testlAndJcc(rtmp1, config.shenandoahGCStateHasForwarded, AMD64Assembler.ConditionFlag.NotZero, csetCheck, false); + } + masm.bind(done); + + // Check for object in collection set in an out-of-line mid-path. + if (strength == ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG) { + crb.getLIR().addSlowPath(op, () -> { + masm.bind(csetCheck); + + masm.movq(rtmp1, HotSpotReplacementsUtil.shenandoahGCCSetFastTestAddr(config)); + masm.movq(rtmp2, objectRegister); + masm.shrq(rtmp2, HotSpotReplacementsUtil.shenandoahGCRegionSizeBytesShift(config)); + + masm.addq(rtmp2, rtmp1); + masm.cmpb(new AMD64Address(rtmp2), 0); + masm.jcc(AMD64Assembler.ConditionFlag.NotZero, slowPath); + + masm.jmp(done); + }); + } + + // Call runtime slow-path LRB in out-of-line slow-path. + crb.getLIR().addSlowPath(op, () -> { + masm.bind(slowPath); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 2, "Expecting callTarget to have only 2 parameters. It has %d", cc.getArgumentCount()); + + AMD64Address cArg0 = (AMD64Address) crb.asAddress(cc.getArgument(0)); + AMD64Address cArg1 = (AMD64Address) crb.asAddress(cc.getArgument(1)); + + // Store first argument + masm.movq(cArg0, objectRegister); + + // Store second argument + masm.leaq(rtmp1, loadAddr); + masm.movq(cArg1, rtmp1); + + // Make the call + AMD64Call.directCall(crb, masm, callTarget, null, false, null); + + // Retrieve result and move to the result register. + AMD64Address cRet = (AMD64Address) crb.asAddress(cc.getReturn()); + masm.movq(resultRegister, cRet); + masm.jmp(done); + }); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java new file mode 100644 index 000000000000..da78a5c92432 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/shenandoah/AMD64HotSpotShenandoahSATBBarrierOp.java @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.hotspot.amd64.shenandoah; + +import jdk.graal.compiler.asm.Label; +import jdk.graal.compiler.asm.amd64.AMD64Address; +import jdk.graal.compiler.asm.amd64.AMD64Assembler; +import jdk.graal.compiler.asm.amd64.AMD64BaseAssembler; +import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; +import jdk.graal.compiler.core.common.CompressEncoding; +import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; +import jdk.graal.compiler.hotspot.amd64.AMD64HotSpotMacroAssembler; +import jdk.graal.compiler.hotspot.meta.HotSpotProviders; +import jdk.graal.compiler.hotspot.replacements.HotSpotReplacementsUtil; +import jdk.graal.compiler.lir.LIRInstructionClass; +import jdk.graal.compiler.lir.SyncPort; +import jdk.graal.compiler.lir.amd64.AMD64Call; +import jdk.graal.compiler.lir.amd64.AMD64LIRInstruction; +import jdk.graal.compiler.lir.amd64.AMD64Move; +import jdk.graal.compiler.lir.asm.CompilationResultBuilder; +import jdk.vm.ci.code.CallingConvention; +import jdk.vm.ci.code.Register; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.asm.Assembler.guaranteeDifferentRegisters; +import static jdk.graal.compiler.core.common.GraalOptions.AssemblyGCBarriersSlowPathOnly; +import static jdk.graal.compiler.core.common.GraalOptions.VerifyAssemblyGCBarriers; +import static jdk.vm.ci.code.ValueUtil.asRegister; + +/** + * X86 backend for the Shenandoah SATB barrier. + */ +public class AMD64HotSpotShenandoahSATBBarrierOp extends AMD64LIRInstruction { + public static final LIRInstructionClass TYPE = LIRInstructionClass.create(AMD64HotSpotShenandoahSATBBarrierOp.class); + + private final GraalHotSpotVMConfig config; + private final HotSpotProviders providers; + + /** + * The SATB slow-path runtime entry. + */ + private final ForeignCallLinkage callTarget; + + /** + * If we know that the previous value is not null, then we don't need to emit a null-check. + */ + private final boolean nonNull; + + /** + * The store address. + */ + @Alive private Value address; + + /** + * The pre-loaded previous value, if any. + */ + @Alive({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value expectedObject; + + @Temp private Value temp; + + @Temp({OperandFlag.REG, OperandFlag.ILLEGAL}) private Value temp2; + + @Temp private Value temp3; + + public AMD64HotSpotShenandoahSATBBarrierOp(GraalHotSpotVMConfig config, HotSpotProviders providers, + AllocatableValue address, AllocatableValue expectedObject, + AllocatableValue temp, AllocatableValue temp2, AllocatableValue temp3, + ForeignCallLinkage callTarget, boolean nonNull) { + super(TYPE); + this.config = config; + this.providers = providers; + this.address = address; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) ^ temp2.equals(Value.ILLEGAL), "only one register is necessary"); + this.expectedObject = expectedObject; + this.temp = temp; + this.temp2 = temp2; + this.temp3 = temp3; + this.callTarget = callTarget; + this.nonNull = nonNull; + GraalError.guarantee(expectedObject.equals(Value.ILLEGAL) || expectedObject.getPlatformKind().getSizeInBytes() == 8, "expected uncompressed pointer"); + } + + public void loadObject(AMD64MacroAssembler masm, Register preVal, Register immediateAddress) { + if (config.useCompressedOops) { + masm.movl(preVal, new AMD64Address(immediateAddress)); + CompressEncoding encoding = config.getOopEncoding(); + AMD64Move.UncompressPointerOp.emitUncompressCode(masm, preVal, encoding.getShift(), providers.getRegisters().getHeapBaseRegister(), false); + } else { + masm.movq(preVal, new AMD64Address(immediateAddress)); + } + } + + @Override + // @formatter:off + @SyncPort(from = "https://github.com/openjdk/jdk/blob/a2743bab4fd203b0791cf47e617c1a95b05ab3cc/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp#L189-L294", + sha1 = "8fb9ab816c4ee89723e0cf3efb15dcea87128044") + // @formatter:on + public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { + Register storeAddress = asRegister(address); + Register thread = providers.getRegisters().getThreadRegister(); + Register tmp = asRegister(temp); + Register tmp3 = asRegister(temp3); + Register previousValue = expectedObject.equals(Value.ILLEGAL) ? asRegister(temp2) : asRegister(expectedObject); + guaranteeDifferentRegisters(storeAddress, thread, tmp, tmp3, previousValue); + + Label done = new Label(); + Label runtime = new Label(); + + // Is marking active? + masm.movb(tmp, new AMD64Address(thread, HotSpotReplacementsUtil.shenandoahGCStateOffset(config))); + masm.testlAndJcc(tmp, config.shenandoahGCStateMarking, AMD64Assembler.ConditionFlag.Zero, done, true); + + // Do we need to load the previous value? + if (expectedObject.equals(Value.ILLEGAL)) { + loadObject(masm, previousValue, storeAddress); + } + + if (!nonNull) { + // Is the previous value null? + masm.testAndJcc(AMD64BaseAssembler.OperandSize.QWORD, previousValue, previousValue, AMD64Assembler.ConditionFlag.Zero, done, true); + } + + if (VerifyAssemblyGCBarriers.getValue(crb.getOptions())) { + verifyOop(masm, previousValue, tmp, tmp3); + } + + if (AssemblyGCBarriersSlowPathOnly.getValue(crb.getOptions())) { + masm.jmp(runtime); + } else { + int satbQueueIndexOffset = HotSpotReplacementsUtil.shenandoahSATBIndexOffset(config); + AMD64Address satbQueueIndex = new AMD64Address(thread, satbQueueIndexOffset); + // tmp := *index_adr + // if tmp == 0 then goto runtime + masm.movq(tmp, satbQueueIndex); + masm.cmpq(tmp, 0); + masm.jcc(AMD64Assembler.ConditionFlag.Equal, runtime); + + // tmp := tmp - wordSize + // *index_adr := tmp + masm.subq(tmp, 8); + masm.movq(satbQueueIndex, tmp); + + // tmp := tmp + *buffer_adr + int satbQueueBufferOffset = HotSpotReplacementsUtil.shenandoahSATBBufferOffset(config); + AMD64Address satbQueueBuffer = new AMD64Address(thread, satbQueueBufferOffset); + masm.movq(tmp3, satbQueueBuffer); + masm.addq(tmp, tmp3); + + // Record the previous value + masm.movq(new AMD64Address(tmp), previousValue); + } + masm.bind(done); + + // Out of line slow path + crb.getLIR().addSlowPath(this, () -> { + masm.bind(runtime); + CallingConvention cc = callTarget.getOutgoingCallingConvention(); + GraalError.guarantee(cc.getArgumentCount() == 1, "Expecting callTarget to have only 1 parameters. It has %d", cc.getArgumentCount()); + + AMD64Address cArg0 = (AMD64Address) crb.asAddress(cc.getArgument(0)); + masm.movq(cArg0, previousValue); + AMD64Call.directCall(crb, masm, callTarget, null, false, null); + masm.jmp(done); + }); + } + + private static void verifyOop(AMD64MacroAssembler masm, Register previousValue, Register tmp, Register tmp2) { + ((AMD64HotSpotMacroAssembler) masm).verifyOop(previousValue, tmp, tmp2, false, true); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java index a3769a5f0254..f3d951338704 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/DefaultHotSpotLoweringProvider.java @@ -792,12 +792,27 @@ private void lowerKlassLayoutHelperNode(KlassLayoutHelperNode n, LoweringTool to } StructuredGraph graph = n.graph(); assert !n.getHub().isConstant(); - AddressNode address = createOffsetAddress(graph, n.getHub(), runtime.getVMConfig().klassLayoutHelperOffset); - ReadNode memoryRead = graph.add(new ReadNode(address, HotSpotReplacementsUtil.KLASS_LAYOUT_HELPER_LOCATION, null, n.stamp(NodeView.DEFAULT), null, BarrierType.NONE)); - graph.addAfterFixed(tool.lastFixedNode(), memoryRead); + ValueNode memoryRead = createRead(graph, createOffsetAddress(graph, n.getHub(), runtime.getVMConfig().klassLayoutHelperOffset), HotSpotReplacementsUtil.KLASS_LAYOUT_HELPER_LOCATION, + n.stamp(NodeView.DEFAULT), BarrierType.NONE, tool.lastFixedNode()); n.replaceAtUsagesAndDelete(memoryRead); } + private static ValueNode createRead(StructuredGraph graph, AddressNode address, LocationIdentity location, Stamp stamp, BarrierType barrierType, FixedWithNextNode insertAfter) { + return createRead(graph, address, location, stamp, null, barrierType, insertAfter); + } + + private static ValueNode createRead(StructuredGraph graph, AddressNode address, LocationIdentity location, Stamp stamp, GuardingNode guard, BarrierType barrierType, + FixedWithNextNode insertAfter) { + if (graph.getGraphState().allowsFloatingReads()) { + return graph.unique(new FloatingReadNode(address, location, null, stamp, guard, barrierType)); + } else { + ReadNode memoryRead = graph.add(new ReadNode(address, location, null, stamp, guard, barrierType)); + graph.addAfterFixed(insertAfter, memoryRead); + assert memoryRead.canFloat() : "this path is only useable for floatable reads: " + memoryRead; + return memoryRead; + } + } + private void lowerHubGetClassNode(HubGetClassNode n, LoweringTool tool) { if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.HIGH_TIER) { return; @@ -809,13 +824,16 @@ private void lowerHubGetClassNode(HubGetClassNode n, LoweringTool tool) { assert !hub.isConstant(); AddressNode mirrorAddress = createOffsetAddress(graph, hub, vmConfig.classMirrorOffset); Stamp loadStamp = n.stamp(NodeView.DEFAULT); - FloatingReadNode read = graph.unique( - new FloatingReadNode(mirrorAddress, HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION, null, StampFactory.forKind(target.wordJavaKind), - null, BarrierType.NONE)); + FixedWithNextNode insertAfter = tool.lastFixedNode(); + ValueNode read = createRead(graph, mirrorAddress, HotSpotReplacementsUtil.CLASS_MIRROR_LOCATION, StampFactory.forKind(target.wordJavaKind), + BarrierType.NONE, insertAfter); + if (read instanceof FixedWithNextNode fixed) { + insertAfter = fixed; + } // Read the Object from the OopHandle AddressNode address = createOffsetAddress(graph, read, 0); - read = graph.unique(new FloatingReadNode(address, HotSpotReplacementsUtil.HOTSPOT_OOP_HANDLE_LOCATION, null, loadStamp, null, - barrierSet.readBarrierType(HotSpotReplacementsUtil.HOTSPOT_OOP_HANDLE_LOCATION, address, loadStamp))); + read = createRead(graph, address, HotSpotReplacementsUtil.HOTSPOT_OOP_HANDLE_LOCATION, loadStamp, + barrierSet.readBarrierType(HotSpotReplacementsUtil.HOTSPOT_OOP_HANDLE_LOCATION, address, loadStamp), insertAfter); n.replaceAtUsagesAndDelete(read); } @@ -827,7 +845,7 @@ private void lowerClassGetHubNode(ClassGetHubNode n, LoweringTool tool) { StructuredGraph graph = n.graph(); assert !n.getValue().isConstant(); AddressNode address = createOffsetAddress(graph, n.getValue(), runtime.getVMConfig().klassOffset); - FloatingReadNode read = graph.unique(new FloatingReadNode(address, HotSpotReplacementsUtil.CLASS_KLASS_LOCATION, null, n.stamp(NodeView.DEFAULT), null, BarrierType.NONE)); + ValueNode read = createRead(graph, address, HotSpotReplacementsUtil.CLASS_KLASS_LOCATION, n.stamp(NodeView.DEFAULT), BarrierType.NONE, tool.lastFixedNode()); n.replaceAtUsagesAndDelete(read); } @@ -920,9 +938,7 @@ protected ValueNode createReadArrayComponentHub(StructuredGraph graph, ValueNode guard = AbstractBeginNode.prevBegin(anchor); } AddressNode address = createOffsetAddress(graph, arrayHub, runtime.getVMConfig().arrayClassElementOffset); - ReadNode read = graph.add(new ReadNode(address, HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION, null, KlassPointerStamp.klassNonNull(), guard, BarrierType.NONE)); - graph.addAfterFixed(insertAfter, read); - return read; + return createRead(graph, address, HotSpotReplacementsUtil.OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION, KlassPointerStamp.klassNonNull(), guard, BarrierType.NONE, insertAfter); } private void lowerLoadMethodNode(LoadMethodNode loadMethodNode) { @@ -1157,13 +1173,9 @@ protected ValueNode createReadHub(StructuredGraph graph, ValueNode object, Lower hubStamp = hubStamp.compressed(config.getKlassEncoding()); } - FixedWithNextNode effectiveInsertAfter = insertAfter; - if (config.useCompactObjectHeaders) { AddressNode address = createOffsetAddress(graph, object, config.markOffset); - ReadNode memoryRead = graph.add(new ReadNode(address, COMPACT_HUB_LOCATION, null, StampFactory.forKind(JavaKind.Long), null, BarrierType.NONE)); - graph.addAfterFixed(insertAfter, memoryRead); - effectiveInsertAfter = memoryRead; + ValueNode memoryRead = createRead(graph, address, COMPACT_HUB_LOCATION, StampFactory.forKind(JavaKind.Long), BarrierType.NONE, insertAfter); ValueNode rawCompressedHubWordSize = graph.addOrUnique(UnsignedRightShiftNode.create(memoryRead, ConstantNode.forInt(config.markWordKlassShift, graph), NodeView.DEFAULT)); ValueNode rawCompressedHub = graph.addOrUnique(NarrowNode.create(rawCompressedHubWordSize, 32, NodeView.DEFAULT)); ValueNode compressedKlassPointer = graph.addOrUnique(PointerCastNode.create(hubStamp, rawCompressedHub)); @@ -1171,8 +1183,7 @@ protected ValueNode createReadHub(StructuredGraph graph, ValueNode object, Lower } AddressNode address = createOffsetAddress(graph, object, config.hubOffset); LocationIdentity hubLocation = config.useCompressedClassPointers ? COMPRESSED_HUB_LOCATION : HUB_LOCATION; - ReadNode memoryRead = graph.add(new ReadNode(address, hubLocation, null, hubStamp, null, BarrierType.NONE)); - graph.addAfterFixed(effectiveInsertAfter, memoryRead); + ValueNode memoryRead = createRead(graph, address, hubLocation, hubStamp, BarrierType.NONE, insertAfter); if (config.useCompressedClassPointers) { return HotSpotCompressionNode.uncompress(graph, memoryRead, config.getKlassEncoding()); } else { diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java index 4a1ed947f986..65ea199b8596 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/meta/HotSpotHostForeignCallsProvider.java @@ -80,6 +80,7 @@ import static jdk.graal.compiler.hotspot.HotSpotBackend.VM_ERROR; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.COMPUTES_REGISTERS_KILLED; import static jdk.graal.compiler.hotspot.HotSpotForeignCallLinkage.RegisterEffect.DESTROYS_ALL_CALLER_SAVE_REGISTERS; +import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Shenandoah; import static jdk.graal.compiler.hotspot.HotSpotGraalRuntime.HotSpotGC.Z; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNCOMMON_TRAP; import static jdk.graal.compiler.hotspot.HotSpotHostBackend.DEOPT_BLOB_UNPACK; @@ -238,6 +239,35 @@ public abstract class HotSpotHostForeignCallsProvider extends HotSpotForeignCall public static final HotSpotForeignCallDescriptor Z_ARRAY_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, NO_LOCATIONS, "load_barrier_on_oop_array", void.class, long.class, long.class); + /** + * Shenandoah runtime function entries. + */ + + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_strong_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_strong_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_weak_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_WEAK_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_weak_narrow", Object.class, Object.class, Word.class); + + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom(oopDesc* o, oop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom", Object.class, Object.class, Word.class); + // oopDesc* ShenandoahRuntime::load_reference_barrier_phantom_narrow(oopDesc* o, narrowOop* p); + public static final HotSpotForeignCallDescriptor SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, HAS_SIDE_EFFECT, any(), + "ShenandoahRuntime::load_reference_barrier_phantom_narrow", Object.class, Object.class, Word.class); + + // void ShenandoahRuntime::write_barrier_pre(oopDesc*) + public static final HotSpotForeignCallDescriptor SHENANDOAH_WRITE_BARRIER_PRE = new HotSpotForeignCallDescriptor(LEAF_NO_VZERO, NO_SIDE_EFFECT, NO_LOCATIONS, + "ShenandoahRuntime::write_barrier_pre_stack_only", void.class, Object.class); + /** * Signature of an unsafe {@link System#arraycopy} stub. * @@ -546,6 +576,14 @@ public void initialize(HotSpotProviders providers, OptionValues options) { linkStackOnlyForeignCall(options, providers, G1WBPRECALL_STACK_ONLY, c.writeBarrierPreAddress, PREPEND_THREAD); linkStackOnlyForeignCall(options, providers, G1WBPOSTCALL_STACK_ONLY, c.writeBarrierPostAddress, PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER, c.shenandoahLoadBarrierStrong, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_NARROW, c.shenandoahLoadBarrierStrongNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK, c.shenandoahLoadBarrierWeak, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_WEAK_NARROW, c.shenandoahLoadBarrierWeakNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM, c.shenandoahLoadBarrierPhantom, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_LOAD_BARRIER_PHANTOM_NARROW, c.shenandoahLoadBarrierPhantomNarrow, DONT_PREPEND_THREAD); + linkStackOnlyForeignCall(c.gc == Shenandoah, options, providers, SHENANDOAH_WRITE_BARRIER_PRE, c.shenandoahWriteBarrierPre, DONT_PREPEND_THREAD); + linkForeignCall(options, providers, LOG_PRINTF, c.logPrintfAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_OBJECT, c.logObjectAddress, PREPEND_THREAD); linkForeignCall(options, providers, LOG_PRIMITIVE, c.logPrimitiveAddress, PREPEND_THREAD); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java index 5cf32565dbff..72ec01582775 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/nodes/HotSpotCompressionNode.java @@ -63,6 +63,14 @@ public static CompressionNode uncompress(StructuredGraph graph, ValueNode input, return graph.unique(uncompress(input, encoding)); } + public static HotSpotCompressionNode compressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(compress(input, encoding)); + } + + public static CompressionNode uncompressWithoutUnique(StructuredGraph graph, ValueNode input, CompressEncoding encoding) { + return graph.addWithoutUnique(uncompress(input, encoding)); + } + private static HotSpotCompressionNode compress(ValueNode input, CompressEncoding encoding) { return new HotSpotCompressionNode(CompressionOp.Compress, input, encoding); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/AssertionSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/AssertionSnippets.java index f893f0cc4709..6d986a0e25cf 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/AssertionSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/AssertionSnippets.java @@ -87,7 +87,7 @@ public Templates(OptionValues options, HotSpotProviders providers) { public void lower(AssertionNode assertionNode, LoweringTool tool) { StructuredGraph graph = assertionNode.graph(); - Arguments args = new Arguments(graph.start() instanceof StubStartNode ? stubAssertion : assertion, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(graph.start() instanceof StubStartNode ? stubAssertion : assertion, graph, tool.getLoweringStage()); args.add("condition", assertionNode.condition()); args.add("message", graph.unique(new ConstantNode(new CStringConstant("failed runtime assertion in snippet/stub: " + assertionNode.message() + " (" + graph.method() + ")"), diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotAllocationSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotAllocationSnippets.java index 8eb112d39175..210c1a30c309 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotAllocationSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotAllocationSnippets.java @@ -666,7 +666,7 @@ public void lower(NewInstanceNode node, LoweringTool tool) { long size = type.instanceSize(); OptionValues localOptions = graph.getOptions(); - Arguments args = new Arguments(allocateInstance, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstance, graph, tool.getLoweringStage()); args.add("hub", hub); // instanceSize returns a negative number for types which should be slow path allocated args.add("size", NumUtil.safeAbs(size)); @@ -689,7 +689,7 @@ public void lower(NewInstanceWithExceptionNode node, LoweringTool tool) { long size = type.instanceSize(); OptionValues localOptions = graph.getOptions(); - Arguments args = new Arguments(allocateInstance, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstance, graph, tool.getLoweringStage()); args.add("hub", hub); // instanceSize returns a negative number for types which should be slow path allocated args.add("size", NumUtil.safeAbs(size)); @@ -717,7 +717,7 @@ public void lower(NewArrayNode node, LoweringTool tool) { int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind)); OptionValues localOptions = graph.getOptions(); - Arguments args = new Arguments(allocateArray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArray, graph, tool.getLoweringStage()); args.add("hub", hub); ValueNode length = node.length(); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); @@ -747,7 +747,7 @@ public void lower(NewArrayWithExceptionNode node, LoweringTool tool) { int log2ElementSize = CodeUtil.log2(tool.getMetaAccess().getArrayIndexScale(elementKind)); OptionValues localOptions = graph.getOptions(); - Arguments args = new Arguments(allocateArray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArray, graph, tool.getLoweringStage()); args.add("hub", hub); ValueNode length = node.length(); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); @@ -777,7 +777,7 @@ public void lower(NewMultiArrayNode node, LoweringTool tool) { HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) node.type(); ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), tool.getMetaAccess(), graph); - Arguments args = new Arguments(newmultiarray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(newmultiarray, graph, tool.getLoweringStage()); args.add("hub", hub); args.add("rank", rank); args.add("withException", false); @@ -796,7 +796,7 @@ public void lower(NewMultiArrayWithExceptionNode node, LoweringTool tool) { HotSpotResolvedObjectType type = (HotSpotResolvedObjectType) node.type(); ConstantNode hub = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), type.klass(), tool.getMetaAccess(), graph); - Arguments args = new Arguments(newmultiarray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(newmultiarray, graph, tool.getLoweringStage()); args.add("hub", hub); args.add("rank", rank); args.add("withException", true); @@ -808,7 +808,7 @@ public void lower(NewMultiArrayWithExceptionNode node, LoweringTool tool) { public void lower(DynamicNewInstanceNode node, LoweringTool tool) { OptionValues localOptions = node.graph().getOptions(); - Arguments args = new Arguments(allocateInstanceDynamic, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstanceDynamic, node.graph(), tool.getLoweringStage()); args.add("type", node.getInstanceType()); args.add("fillContents", FillContent.fromBoolean(node.fillContents())); args.add("emitMemoryBarrier", node.emitMemoryBarrier()); @@ -821,7 +821,7 @@ public void lower(DynamicNewInstanceNode node, LoweringTool tool) { public void lower(DynamicNewInstanceWithExceptionNode node, LoweringTool tool) { OptionValues localOptions = node.graph().getOptions(); - Arguments args = new Arguments(allocateInstanceDynamic, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstanceDynamic, node.graph(), tool.getLoweringStage()); args.add("type", node.getInstanceType()); args.add("fillContents", FillContent.fromBoolean(true)); args.add("emitMemoryBarrier", true/* barriers */); @@ -834,7 +834,7 @@ public void lower(DynamicNewInstanceWithExceptionNode node, LoweringTool tool) { public void lower(ValidateNewInstanceClassNode node, LoweringTool tool) { StructuredGraph graph = node.graph(); - Arguments args = new Arguments(validateNewInstanceClass, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(validateNewInstanceClass, graph, tool.getLoweringStage()); args.add("type", node.getInstanceType()); args.add("classClass", node.getClassClass()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, DEFAULT_REPLACER, args); @@ -847,7 +847,7 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) { ValueNode voidClass = node.getVoidClass(); assert voidClass != null; - Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArrayDynamic, graph, tool.getLoweringStage()); args.add("elementType", node.getElementType()); args.add("voidClass", voidClass); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); @@ -878,7 +878,7 @@ public void lower(DynamicNewArrayWithExceptionNode node, LoweringTool tool) { ValueNode voidClass = node.getVoidClass(); assert voidClass != null; - Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArrayDynamic, graph, tool.getLoweringStage()); args.add("elementType", node.getElementType()); args.add("voidClass", voidClass); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); @@ -901,7 +901,7 @@ public void lower(DynamicNewArrayWithExceptionNode node, LoweringTool tool) { public void lower(VerifyHeapNode node, LoweringTool tool) { if (config.cAssertions) { - Arguments args = new Arguments(verifyHeap, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(verifyHeap, node.graph(), tool.getLoweringStage()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, DEFAULT_REPLACER, args); } else { @@ -910,14 +910,14 @@ public void lower(VerifyHeapNode node, LoweringTool tool) { } public void lower(KlassBeingInitializedCheckNode node, LoweringTool tool) { - Arguments args = new Arguments(threadBeingInitializedCheck, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(threadBeingInitializedCheck, node.graph(), tool.getLoweringStage()); args.add("klass", node.getKlass()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, DEFAULT_REPLACER, args); } public void lower(KlassFullyInitializedCheckNode node, LoweringTool tool) { - Arguments args = new Arguments(klassFullyInitializedCheck, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(klassFullyInitializedCheck, node.graph(), tool.getLoweringStage()); args.add("klass", node.getKlass()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, DEFAULT_REPLACER, args); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java index 8e2a11945e6b..99b63e917cec 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/HotSpotReplacementsUtil.java @@ -681,6 +681,36 @@ public static boolean useCondCardMark(@InjectedParameter GraalHotSpotVMConfig co return config.useCondCardMark; } + @Fold + public static int shenandoahGCStateOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCStateOffset; + } + + @Fold + public static int shenandoahSATBIndexOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBIndexOffset; + } + + @Fold + public static int shenandoahSATBBufferOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahSATBBufferOffset; + } + + @Fold + public static int shenandoahCardTableOffset(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahCardTableOffset; + } + + @Fold + public static int shenandoahGCRegionSizeBytesShift(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCRegionSizeBytesShift; + } + + @Fold + public static long shenandoahGCCSetFastTestAddr(@InjectedParameter GraalHotSpotVMConfig config) { + return config.shenandoahGCCSetFastTestAddress; + } + public static final LocationIdentity KLASS_SUPER_CHECK_OFFSET_LOCATION = NamedLocationIdentity.immutable("Klass::_super_check_offset"); @Fold diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java index bd449e31a2fb..8c627f76f6f7 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/InstanceOfSnippets.java @@ -293,22 +293,22 @@ protected Arguments makeArguments(InstanceOfUsageReplacer replacer, LoweringTool StructuredGraph graph = instanceOf.graph(); if (hintInfo.hintHitProbability >= 1.0 && hintInfo.exact == null) { Hints hints = createHints(hintInfo, tool.getMetaAccess(), false, graph); - args = new Arguments(instanceofWithProfile, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(instanceofWithProfile, graph, tool.getLoweringStage()); args.add("object", object); args.addVarargs("hints", KlassPointer.class, KlassPointerStamp.klassNonNull(), hints.hubs); args.addVarargs("hintIsPositive", boolean.class, StampFactory.forKind(JavaKind.Boolean), hints.isPositive); } else if (hintInfo.exact != null) { - args = new Arguments(instanceofExact, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(instanceofExact, graph, tool.getLoweringStage()); args.add("object", object); args.add("exactHub", ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), ((HotSpotResolvedObjectType) hintInfo.exact).klass(), tool.getMetaAccess(), graph)); } else if (type.isPrimaryType()) { - args = new Arguments(instanceofPrimary, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(instanceofPrimary, graph, tool.getLoweringStage()); args.add("hub", hub); args.add("object", object); args.add("superCheckOffset", type.superCheckOffset()); } else { Hints hints = createHints(hintInfo, tool.getMetaAccess(), false, graph); - args = new Arguments(instanceofSecondary, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(instanceofSecondary, graph, tool.getLoweringStage()); args.add("hub", hub); args.add("object", object); args.addVarargs("hints", KlassPointer.class, KlassPointerStamp.klassNonNull(), hints.hubs); @@ -326,7 +326,7 @@ protected Arguments makeArguments(InstanceOfUsageReplacer replacer, LoweringTool InstanceOfDynamicNode instanceOf = (InstanceOfDynamicNode) replacer.instanceOf; ValueNode object = instanceOf.getObject(); - Arguments args = new Arguments(instanceofDynamic, instanceOf.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(instanceofDynamic, instanceOf.graph(), tool.getLoweringStage()); args.add("hub", instanceOf.getMirrorOrHub()); args.add("object", object); args.add("trueValue", replacer.trueValue); @@ -337,7 +337,7 @@ protected Arguments makeArguments(InstanceOfUsageReplacer replacer, LoweringTool return args; } else if (replacer.instanceOf instanceof ClassIsAssignableFromNode) { ClassIsAssignableFromNode isAssignable = (ClassIsAssignableFromNode) replacer.instanceOf; - Arguments args = new Arguments(isAssignableFrom, isAssignable.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(isAssignableFrom, isAssignable.graph(), tool.getLoweringStage()); assert ((ObjectStamp) isAssignable.getThisClass().stamp(NodeView.DEFAULT)).nonNull(); assert ((ObjectStamp) isAssignable.getOtherClass().stamp(NodeView.DEFAULT)).nonNull(); args.add("thisClassNonNull", isAssignable.getThisClass()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LoadExceptionObjectSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LoadExceptionObjectSnippets.java index 152cc0152039..1d239eb1ab8f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LoadExceptionObjectSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LoadExceptionObjectSnippets.java @@ -105,7 +105,7 @@ public void lower(LoadExceptionObjectNode loadExceptionObject, HotSpotRegistersP loadExceptionC.computeStateDuring(loadExceptionObject.stateAfter()); graph.replaceFixedWithFixed(loadExceptionObject, loadExceptionC); } else { - Arguments args = new Arguments(loadException, loadExceptionObject.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(loadException, loadExceptionObject.graph(), tool.getLoweringStage()); args.add("threadRegister", registers.getThreadRegister()); template(tool, loadExceptionObject, args).instantiate(tool.getMetaAccess(), loadExceptionObject, DEFAULT_REPLACER, args); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LogSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LogSnippets.java index d94ce63db446..a9c211c21a3f 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LogSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/LogSnippets.java @@ -94,7 +94,7 @@ public void lower(LogNode logNode, LoweringTool tool) { } else if (logNode.getL1() != null) { info = printf1; } - Arguments args = new Arguments(info, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(info, graph, tool.getLoweringStage()); args.add("message", graph.unique(new ConstantNode(new CStringConstant(logNode.message()), StampFactory.pointer()))); if (logNode.getL1() != null) { args.add("l1", logNode.getL1()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java index b16a7a9e24ed..5b2c15f91ad4 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java @@ -686,7 +686,7 @@ public Templates(OptionValues options, SnippetCounter.Group.Factory factory, Hot public void lower(CheckFastPathMonitorEnterNode checkFastPathMonitorEnterNode, HotSpotRegistersProvider registers, LoweringTool tool) { StructuredGraph graph = checkFastPathMonitorEnterNode.graph(); - Arguments args = new Arguments(checkMonitorenter, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(checkMonitorenter, graph, tool.getLoweringStage()); // Speculation.equals is too weak so it can incorrectly cache snippet graphs so just // disable caching of these graphs. args.setCacheable(false); @@ -744,7 +744,7 @@ public void lower(MonitorEnterNode monitorenterNode, HotSpotRegistersProvider re GraalError.guarantee(!monitorenterNode.getMonitorId().isEliminated(), "current monitor is eliminated: %s", monitorenterNode); GraalError.guarantee(verifyLockOrder(monitorenterNode), "locks are disordered: %s", monitorenterNode); - Arguments args = new Arguments(monitorenter, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(monitorenter, graph, tool.getLoweringStage()); args.add("object", monitorenterNode.object()); args.add("hub", Objects.requireNonNull(monitorenterNode.getObjectData())); args.add("lockDepth", monitorenterNode.getMonitorId().getLockDepth()); @@ -760,7 +760,7 @@ public void lower(MonitorEnterNode monitorenterNode, HotSpotRegistersProvider re public void lower(MonitorExitNode monitorexitNode, HotSpotRegistersProvider registers, LoweringTool tool) { StructuredGraph graph = monitorexitNode.graph(); - Arguments args = new Arguments(monitorexit, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(monitorexit, graph, tool.getLoweringStage()); args.add("object", monitorexitNode.object()); args.add("lockDepth", monitorexitNode.getMonitorId().getLockDepth()); args.add("threadRegister", registers.getThreadRegister()); @@ -835,7 +835,7 @@ private void checkBalancedMonitors(StructuredGraph graph, LoweringTool tool) { invoke.setStateAfter(graph.add(stateAfter)); graph.addBeforeFixed(ret, invoke); - Arguments args = new Arguments(checkCounter, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(checkCounter, graph, tool.getLoweringStage()); args.add("errMsg", new CStringConstant(msg)); inlineeGraph = template(tool, invoke, args).copySpecializedGraph(graph.getDebug()); InliningUtil.inline(invoke, inlineeGraph, false, null); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/ObjectSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/ObjectSnippets.java index 78b5b1b5a51b..b3ea9c54d41b 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/ObjectSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/ObjectSnippets.java @@ -111,7 +111,7 @@ public void lower(Node n, LoweringTool tool) { StructuredGraph graph = (StructuredGraph) n.graph(); FrameState stateDuringCall = fn.stateDuring(); assert stateDuringCall != null : "Must have valid state for snippet recursive notify call"; - Arguments args = new Arguments(fn.isNotifyAll() ? notifyAllSnippet : notifySnippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(fn.isNotifyAll() ? notifyAllSnippet : notifySnippet, graph, tool.getLoweringStage()); args.add("thisObj", fn.object); SnippetTemplate template = template(tool, fn, args); graph.getDebug().log("Lowering fast notify in %s: node=%s, template=%s, arguments=%s", graph, fn, template, args); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/RegisterFinalizerSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/RegisterFinalizerSnippets.java index c2ada6736ac4..80d5454651ab 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/RegisterFinalizerSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/RegisterFinalizerSnippets.java @@ -78,7 +78,7 @@ public Templates(OptionValues options, HotSpotProviders providers) { public void lower(RegisterFinalizerNode node, LoweringTool tool) { assert !(node instanceof LoweredRegisterFinalizerNode) : Assertions.errorMessage(node); StructuredGraph graph = node.graph(); - Arguments args = new Arguments(registerFinalizerSnippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(registerFinalizerSnippet, graph, tool.getLoweringStage()); args.add("thisObj", node.getValue()); SnippetTemplate template = template(tool, node, args); template.instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/UnsafeSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/UnsafeSnippets.java index 618e4c903f37..0ddb1e0f10f3 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/UnsafeSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/UnsafeSnippets.java @@ -94,7 +94,7 @@ public Templates(OptionValues options, HotSpotProviders providers) { public void lower(UnsafeCopyMemoryNode copyMemoryNode, LoweringTool tool) { StructuredGraph graph = copyMemoryNode.graph(); - Arguments args = new Arguments(copyMemory, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(copyMemory, graph, tool.getLoweringStage()); args.add("receiver", copyMemoryNode.receiver); args.add("srcBase", copyMemoryNode.srcBase); args.add("srcOffset", copyMemoryNode.srcOffset); @@ -107,7 +107,7 @@ public void lower(UnsafeCopyMemoryNode copyMemoryNode, LoweringTool tool) { public void lower(UnsafeSetMemoryNode setMemoryNode, LoweringTool tool) { StructuredGraph graph = setMemoryNode.graph(); - Arguments args = new Arguments(setMemory, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(setMemory, graph, tool.getLoweringStage()); args.add("receiver", setMemoryNode.receiver); args.add("objBase", setMemoryNode.obj); args.add("objOffset", setMemoryNode.offset); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/VirtualThreadUpdateJFRSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/VirtualThreadUpdateJFRSnippets.java index 2b24c490bfb3..28dd441bc354 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/VirtualThreadUpdateJFRSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/VirtualThreadUpdateJFRSnippets.java @@ -152,7 +152,7 @@ public Templates(OptionValues options, HotSpotProviders providers) { } public void lower(VirtualThreadUpdateJFRNode virtualThreadUpdateJFRNode, HotSpotRegistersProvider registers, LoweringTool tool) { - Arguments args = new Arguments(virtualThreadUpdateJFR, virtualThreadUpdateJFRNode.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(virtualThreadUpdateJFR, virtualThreadUpdateJFRNode.graph(), tool.getLoweringStage()); args.add("javaThreadRegister", registers.getThreadRegister()); args.add("threadObj", virtualThreadUpdateJFRNode.getThread()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java index b91576855f24..fe3520b732ba 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64AtomicMove.java @@ -139,7 +139,7 @@ public void emitCode(CompilationResultBuilder crb, AArch64MacroAssembler masm) { emitCompareAndSwap(masm, accessKind, address, result, expected, asRegister(newValue), memoryOrder, setConditionFlags); } - protected static void emitCompareAndSwap(AArch64MacroAssembler masm, AArch64Kind accessKind, Register address, Register result, Register expected, Register newValue, + public static void emitCompareAndSwap(AArch64MacroAssembler masm, AArch64Kind accessKind, Register address, Register result, Register expected, Register newValue, MemoryOrderMode memoryOrder, boolean setConditionFlags) { assert accessKind.isInteger(); final int memAccessSize = accessKind.getSizeInBytes() * Byte.SIZE; diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java index 5309f24115a4..b685224c4aa0 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64Move.java @@ -986,6 +986,10 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { final Register resReg = getResultRegister(); final Register baseReg = getBaseRegister(); + emitCompressCode(masm, resReg, getShift(), baseReg, nonNull); + } + + public static void emitCompressCode(AMD64MacroAssembler masm, Register resReg, int shift, Register baseReg, boolean nonNull) { if (!baseReg.equals(Register.None)) { if (!nonNull) { masm.testq(resReg, resReg); @@ -994,7 +998,6 @@ public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) { masm.subq(resReg, baseReg); } - int shift = getShift(); if (shift != 0) { masm.shrq(resReg, shift); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java new file mode 100644 index 000000000000..257b3c85e024 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/gen/ShenandoahBarrierSetLIRGeneratorTool.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.lir.gen; + +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +/** + * Lowers Shenandoah barriers to LIR. + */ +public interface ShenandoahBarrierSetLIRGeneratorTool extends BarrierSetLIRGeneratorTool { + Value emitLoadReferenceBarrier(LIRGeneratorTool tool, Value obj, Value address, ShenandoahLoadRefBarrierNode.ReferenceStrength strength, boolean narrow, boolean notNull); + + void emitPreWriteBarrier(LIRGeneratorTool lirTool, Value address, AllocatableValue expectedObject, boolean nonNull); + + void emitCardBarrier(LIRGeneratorTool lirTool, Value address); +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java index b2c2479e9cc9..d3dfa59287de 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/GraphState.java @@ -68,7 +68,8 @@ public final class GraphState { StageFlag.EXPAND_LOGIC, StageFlag.ADDRESS_LOWERING, StageFlag.REMOVE_OPAQUE_VALUES, - StageFlag.FINAL_SCHEDULE); + StageFlag.FINAL_SCHEDULE, + StageFlag.LOW_TIER_BARRIER_ADDITION); private static final EnumSet ENTERPRISE_MID_TIER_MANDATORY_STAGES = EnumSet.of( StageFlag.OPTIMISTIC_ALIASING, StageFlag.GUARD_LOWERING, @@ -561,6 +562,10 @@ public boolean isExplicitExceptionsNoDeopt() { return guardsStage == GuardsStage.FIXED_DEOPTS && isAfterStage(StageFlag.GUARD_LOWERING); } + public boolean allowsFloatingReads() { + return isAfterStage(StageFlag.FLOATING_READS) && isBeforeStage(StageFlag.FIXED_READS); + } + /** * Configure the graph to only allow explicit exception edges without floating guard nodes. That * is the graph: @@ -632,6 +637,7 @@ public enum StageFlag { VECTOR_LOWERING, EXPAND_LOGIC, FIXED_READS, + LOW_TIER_BARRIER_ADDITION, PARTIAL_REDUNDANCY_SCHEDULE, ADDRESS_LOWERING, FINAL_CANONICALIZATION, diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java index 86871a540585..4118cd123155 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/BarrierSet.java @@ -25,6 +25,7 @@ */ package jdk.graal.compiler.nodes.gc; +import jdk.graal.compiler.nodes.GraphState; import org.graalvm.word.LocationIdentity; import jdk.graal.compiler.core.common.memory.BarrierType; @@ -88,4 +89,10 @@ default BarrierType writeBarrierType(LocationIdentity location) { */ default void verifyBarriers(StructuredGraph graph) { } + + default boolean shouldAddBarriersInStage(GraphState.StageFlag stage) { + // Most barrier sets should be added in mid-tier, some might also + // wish to add in low-tier (e.g. Shenandoah GC). + return stage == GraphState.StageFlag.BARRIER_ADDITION; + } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java new file mode 100644 index 000000000000..716680d77d6e --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahBarrierSet.java @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import static jdk.graal.compiler.nodes.NamedLocationIdentity.OFF_HEAP_LOCATION; + +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.nodes.GraphState; +import jdk.graal.compiler.nodes.extended.ArrayRangeWrite; +import jdk.graal.compiler.nodes.gc.BarrierSet; +import jdk.graal.compiler.nodes.java.ValueCompareAndSwapNode; +import jdk.graal.compiler.nodes.spi.CoreProviders; +import jdk.graal.compiler.nodes.type.NarrowOopStamp; +import org.graalvm.word.LocationIdentity; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.Node; +import jdk.graal.compiler.nodes.FieldLocationIdentity; +import jdk.graal.compiler.nodes.NamedLocationIdentity; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.StructuredGraph; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.extended.RawStoreNode; +import jdk.graal.compiler.nodes.java.AbstractCompareAndSwapNode; +import jdk.graal.compiler.nodes.java.LoweredAtomicReadAndWriteNode; +import jdk.graal.compiler.nodes.memory.AddressableMemoryAccess; +import jdk.graal.compiler.nodes.memory.FixedAccessNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.ReadNode; +import jdk.graal.compiler.nodes.memory.WriteNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.type.StampTool; +import jdk.vm.ci.meta.JavaKind; +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Shenandoah barrier set implementation. + * + * This generates 3 kinds of barriers: + * + *
    + *
  • Load-reference barriers after reference-loads. The purpose is to canonicalize references + * during concurrent collection, where we might otherwise see both from- and to-space references to + * the same object.
    • + * + *
  • SATB barriers before reference writes. Those support concurrent marking, similar to how this + * is done in G1. Reference.get-barriers are a special form of this, to support (weak,soft,phantom-) + * references.
    • + * + *
  • Card barriers, only needed for generational Shenandoah. Those are inserted after + * reference-writes and dirty cards. Similar to their counterparts in Serial and Parallel GC.
    • + *
+ */ +public class ShenandoahBarrierSet implements BarrierSet { + + private final ResolvedJavaType objectArrayType; + private final ResolvedJavaField referentField; + protected boolean useLoadRefBarrier; + protected boolean useSATBBarrier; + protected boolean useCASBarrier; + protected boolean useCardBarrier; + + public ShenandoahBarrierSet(ResolvedJavaType objectArrayType, ResolvedJavaField referentField) { + this.referentField = referentField; + this.objectArrayType = objectArrayType; + this.useLoadRefBarrier = true; + this.useSATBBarrier = true; + this.useCASBarrier = true; + this.useCardBarrier = true; + } + + @Override + public BarrierType postAllocationInitBarrier(BarrierType original) { + assert original == BarrierType.FIELD || original == BarrierType.ARRAY : "only for write barriers: " + original; + return BarrierType.POST_INIT_WRITE; + } + + @Override + public BarrierType readBarrierType(LocationIdentity location, ValueNode address, Stamp loadStamp) { + if (location.equals(OFF_HEAP_LOCATION)) { + // Off heap locations are never expected to contain objects + GraalError.guarantee(!loadStamp.isObjectStamp(), "off-heap location not expected to be object: %s", location); + return BarrierType.NONE; + } + + if (loadStamp.isObjectStamp()) { + if (address.stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + + if (address instanceof AddressNode addr) { + if (addr.getBase().stamp(NodeView.DEFAULT).isObjectStamp()) { + // A read of an Object from an Object requires a barrier + return BarrierType.READ; + } + } + // Objects aren't expected to be read from non-heap locations. + throw GraalError.shouldNotReachHere("Unexpected location type " + loadStamp); + } + + GraalError.guarantee(!(location instanceof FieldLocationIdentity fieldLocationIdentity) || fieldLocationIdentity.getField().getJavaKind() != JavaKind.Object, + "must not be a reference location: %s", address); + return BarrierType.NONE; + } + + @Override + public BarrierType writeBarrierType(RawStoreNode store) { + if (store.object().isNullConstant()) { + return BarrierType.NONE; + } + return store.needsBarrier() ? readWriteBarrier(store.object(), store.value()) : BarrierType.NONE; + } + + @Override + public BarrierType fieldReadBarrierType(ResolvedJavaField field, JavaKind storageKind) { + if (field.getJavaKind() == JavaKind.Object && field.equals(referentField)) { + return BarrierType.REFERENCE_GET; + } + if (storageKind.isObject()) { + return BarrierType.READ; + } + return BarrierType.NONE; + } + + @Override + public BarrierType fieldWriteBarrierType(ResolvedJavaField field, JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.FIELD : BarrierType.NONE; + } + + @Override + public BarrierType arrayWriteBarrierType(JavaKind storageKind) { + return storageKind == JavaKind.Object ? BarrierType.ARRAY : BarrierType.NONE; + } + + @Override + public BarrierType readWriteBarrier(ValueNode object, ValueNode value) { + if (value.stamp(NodeView.DEFAULT).isObjectStamp()) { + ResolvedJavaType type = StampTool.typeOrNull(object); + if (type != null && type.isArray()) { + return BarrierType.ARRAY; + } else if (type == null || type.isAssignableFrom(objectArrayType)) { + return BarrierType.ARRAY; + } else { + return BarrierType.FIELD; + } + } + return BarrierType.NONE; + } + + @Override + public boolean hasWriteBarrier() { + return true; + } + + @Override + public boolean hasReadBarrier() { + return true; + } + + @Override + public void addBarriers(FixedAccessNode n, CoreProviders context) { + switch (n) { + case ReadNode readNode -> addReadNodeBarriers(readNode); + case WriteNode write -> addWriteBarriers(write, write.value(), null); + case LoweredAtomicReadAndWriteNode atomic -> { + if (useCASBarrier) { + addWriteBarriers(atomic, atomic.getNewValue(), null); + addReadNodeBarriers(atomic); + } + } + case AbstractCompareAndSwapNode cmpSwap -> { + if (useCASBarrier) { + addWriteBarriers(cmpSwap, cmpSwap.getNewValue(), cmpSwap.getExpectedValue()); + if (cmpSwap instanceof ValueCompareAndSwapNode) { + addReadNodeBarriers(cmpSwap); + } + } + } + case ArrayRangeWrite ignored -> GraalError.unimplemented("ArrayRangeWrite is not used"); + case null, default -> + GraalError.guarantee(n.getBarrierType() == BarrierType.NONE, "missed a node that requires a GC barrier: %s", n.getClass()); + } + } + + private void addWriteBarriers(FixedAccessNode node, ValueNode writtenValue, ValueNode expectedValue) { + BarrierType barrierType = node.getBarrierType(); + switch (barrierType) { + case NONE: + // nothing to do + break; + case FIELD: + case ARRAY: + case UNKNOWN: + case POST_INIT_WRITE: + case AS_NO_KEEPALIVE_WRITE: + if (isObjectValue(writtenValue)) { + StructuredGraph graph = node.graph(); + boolean init = node.getLocationIdentity().isInit(); + if (!init && barrierType != BarrierType.AS_NO_KEEPALIVE_WRITE && useSATBBarrier) { + // The pre barrier does nothing if the value being read is null, so it can + // be explicitly skipped when this is an initializing store. + // No keep-alive means no need for the pre-barrier. + addShenandoahSATBBarrier(node, node.getAddress(), expectedValue, graph); + } + if (!init && useCardBarrier && !StampTool.isPointerAlwaysNull(writtenValue)) { + graph.addAfterFixed(node, graph.add(new ShenandoahCardBarrierNode(node.getAddress()))); + } + } + break; + default: + throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + private void addLoadReferenceBarrier(FixedAccessNode node, AddressNode address, BarrierType barrierType) { + GraalError.guarantee(node != null, "input value must not be null"); + StructuredGraph graph = node.graph(); + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ValueNode uncompressed = maybeUncompressReference(node, narrow); + ShenandoahLoadRefBarrierNode lrb = graph.add(new ShenandoahLoadRefBarrierNode(uncompressed, address, barrierType, narrow)); + ValueNode compValue = maybeCompressReference(lrb, narrow); + ValueNode newUsage = uncompressed != node ? uncompressed : lrb; + node.replaceAtUsages(compValue, usage -> usage != newUsage); + } + + private void addReadNodeBarriers(FixedAccessNode node) { + + BarrierType barrierType = node.getBarrierType(); + StructuredGraph graph = node.graph(); + switch (barrierType) { + case NONE -> { + // No barriers required. + } + case REFERENCE_GET -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + if (useSATBBarrier) { + boolean narrow = node.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahReferentFieldReadBarrierNode barrier = graph.add(new ShenandoahReferentFieldReadBarrierNode(node.getAddress(), maybeUncompressReference(node, narrow))); + graph.addAfterFixed(node, barrier); + } + } + case WEAK_REFERS_TO, PHANTOM_REFERS_TO, READ, ARRAY, FIELD, UNKNOWN -> { + if (useLoadRefBarrier) { + addLoadReferenceBarrier(node, node.getAddress(), barrierType); + } + } + default -> throw new GraalError("unexpected barrier type: " + barrierType); + } + } + + protected ValueNode maybeUncompressReference(ValueNode value, @SuppressWarnings("unused") boolean narrow) { + return value; + } + + protected ValueNode maybeCompressReference(ValueNode value, @SuppressWarnings("unused") boolean narrow) { + return value; + } + + private void addShenandoahSATBBarrier(FixedAccessNode node, AddressNode address, ValueNode value, StructuredGraph graph) { + boolean narrow = value != null && value.stamp(NodeView.DEFAULT) instanceof NarrowOopStamp; + ShenandoahSATBBarrierNode preBarrier = graph.add(new ShenandoahSATBBarrierNode(address, maybeUncompressReference(value, narrow))); + GraalError.guarantee(!node.getUsedAsNullCheck(), "trapping null checks are inserted after write barrier insertion: ", node); + node.setStateBefore(null); + graph.addBeforeFixed(node, preBarrier); + } + + private static boolean isObjectValue(ValueNode value) { + return value.stamp(NodeView.DEFAULT) instanceof AbstractObjectStamp; + } + + @Override + public boolean mayNeedPreWriteBarrier(JavaKind storageKind) { + return false; + } + + @Override + public void verifyBarriers(StructuredGraph graph) { + for (Node node : graph.getNodes()) { + if (node instanceof WriteNode write) { + Stamp stamp = write.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(write.getBarrierType() == BarrierType.NONE, "no barriers for primitive writes: %s", write); + } + } else if (node instanceof ReadNode || + node instanceof AbstractCompareAndSwapNode || + node instanceof LoweredAtomicReadAndWriteNode) { + LIRLowerableAccess read = (LIRLowerableAccess) node; + Stamp stamp = read.getAccessStamp(NodeView.DEFAULT); + if (!stamp.isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barriers for primitive reads: %s", read); + continue; + } + + BarrierType expectedBarrier = barrierForLocation(read.getBarrierType(), read.getLocationIdentity(), JavaKind.Object); + if (expectedBarrier != null) { + GraalError.guarantee(expectedBarrier == read.getBarrierType(), "expected %s but found %s in %s", expectedBarrier, read.getBarrierType(), read); + continue; + } + + ValueNode base = read.getAddress().getBase(); + if (!base.stamp(NodeView.DEFAULT).isObjectStamp()) { + GraalError.guarantee(read.getBarrierType() == BarrierType.NONE, "no barrier for non-heap read: %s", read); + } else { + GraalError.guarantee(read.getBarrierType() == BarrierType.FIELD, "missing barriers for heap read: %s", read); + } + } else if (node instanceof AddressableMemoryAccess access) { + if (access.getBarrierType() != BarrierType.NONE) { + throw new GraalError("Unexpected memory access with barrier : " + node); + } + } + } + } + + protected BarrierType barrierForLocation(BarrierType currentBarrier, LocationIdentity location, JavaKind storageKind) { + if (location instanceof FieldLocationIdentity fieldLocationIdentity) { + BarrierType barrierType = fieldReadBarrierType(fieldLocationIdentity.getField(), storageKind); + if (barrierType != currentBarrier && barrierType == BarrierType.REFERENCE_GET) { + if (currentBarrier == BarrierType.WEAK_REFERS_TO || currentBarrier == BarrierType.PHANTOM_REFERS_TO) { + return currentBarrier; + } + } + return barrierType; + } + if (location.equals(NamedLocationIdentity.getArrayLocation(JavaKind.Object))) { + return BarrierType.READ; + } + return null; + } + + @Override + public boolean shouldAddBarriersInStage(GraphState.StageFlag stage) { + return stage == GraphState.StageFlag.LOW_TIER_BARRIER_ADDITION; + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java new file mode 100644 index 000000000000..94a99444be58 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahCardBarrierNode.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_8; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_4; + +/** + * Shenandoah card barriers. Those are added after reference-writes and serve to dirty cards in the + * card-table. Only needed for generational Shenandoah. + */ +@NodeInfo(cycles = CYCLES_8, size = SIZE_4) +public class ShenandoahCardBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahCardBarrierNode.class); + + public ShenandoahCardBarrierNode(AddressNode address) { + super(TYPE, address, null, false); + } + + @Override + public Kind getKind() { + return Kind.POST_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + tool.emitCardBarrier(gen.getLIRGeneratorTool(), gen.operand(getAddress())); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java new file mode 100644 index 000000000000..f943594cc8ae --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahLoadRefBarrierNode.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.memory.BarrierType; +import jdk.graal.compiler.core.common.type.AbstractObjectStamp; +import jdk.graal.compiler.core.common.type.Stamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.InputType; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.memory.LIRLowerableAccess; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * Shenandoah load-reference barriers. Those are added after reference-loads, and are used to + * canonicalize references during concurrent evacuation. During concurrent evacuation we might see + * both from- and to-space references to the same objects, and this barrier ensures that we only see + * to-space references. (a.k.a. To-space invariant). + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahLoadRefBarrierNode extends ValueNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahLoadRefBarrierNode.class); + + /** + * Strength of the input reference, determines generated code and slow-path call. + */ + public enum ReferenceStrength { + STRONG, + WEAK, + PHANTOM; + } + + /** + * The input value. Typically this is a reference that has just been loaded. The barrier output + * represents the canonicalized reference. + */ + @Input private ValueNode value; + + /** + * The address from which the input value has been loaded, if any/known. + */ + @Input(InputType.Association) private AddressNode address; + + /** + * The strength of the loaded reference. + */ + private final ReferenceStrength strength; + + /** + * Whether the reference is compressed. + */ + private final boolean narrow; + + private static ReferenceStrength getReferenceStrength(BarrierType barrierType) { + return switch (barrierType) { + case READ, FIELD, ARRAY, NONE -> ReferenceStrength.STRONG; + case REFERENCE_GET, WEAK_REFERS_TO -> ReferenceStrength.WEAK; + case PHANTOM_REFERS_TO -> ReferenceStrength.PHANTOM; + case UNKNOWN, POST_INIT_WRITE, AS_NO_KEEPALIVE_WRITE -> throw GraalError.shouldNotReachHere("Unexpected barrier type: " + barrierType); + }; + } + + public ShenandoahLoadRefBarrierNode(ValueNode value, AddressNode address, BarrierType barrierType, boolean narrow) { + super(TYPE, value.stamp(NodeView.DEFAULT)); + this.value = value; + this.address = address; + this.strength = getReferenceStrength(barrierType); + this.narrow = narrow; + } + + @Override + public void generate(NodeLIRBuilderTool gen) { + Stamp valueStamp; + if (value instanceof LIRLowerableAccess accessValue) { + valueStamp = accessValue.getAccessStamp(NodeView.DEFAULT); + } else { + valueStamp = value.stamp(NodeView.DEFAULT); + } + GraalError.guarantee(valueStamp.isObjectStamp(), "LRB value must be object"); + boolean notNull = ((AbstractObjectStamp) valueStamp).nonNull(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) gen.getLIRGeneratorTool().getBarrierSet(); + gen.setResult(this, tool.emitLoadReferenceBarrier(gen.getLIRGeneratorTool(), gen.operand(value), gen.operand(address), strength, narrow, notNull)); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java new file mode 100644 index 000000000000..7504d489e154 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahReferentFieldReadBarrierNode.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * A special case of the SATB barrier, needed to support soft and weak references. They are added + * after reads of referents of SoftReference and WeakReference objects, and ensure that such + * referents are marked live during concurrent marking. + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public class ShenandoahReferentFieldReadBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahReferentFieldReadBarrierNode.class); + + public ShenandoahReferentFieldReadBarrierNode(AddressNode address, ValueNode expectedObject) { + super(TYPE, address, expectedObject, true); + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), lirGen.asAllocatable(generator.operand(getExpectedObject())), false); + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java new file mode 100644 index 000000000000..9aab7bc99c57 --- /dev/null +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/gc/shenandoah/ShenandoahSATBBarrierNode.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.graal.compiler.nodes.gc.shenandoah; + +import jdk.graal.compiler.core.common.type.ObjectStamp; +import jdk.graal.compiler.debug.GraalError; +import jdk.graal.compiler.graph.NodeClass; +import jdk.graal.compiler.lir.gen.LIRGeneratorTool; +import jdk.graal.compiler.lir.gen.ShenandoahBarrierSetLIRGeneratorTool; +import jdk.graal.compiler.nodeinfo.NodeInfo; +import jdk.graal.compiler.nodes.NodeView; +import jdk.graal.compiler.nodes.ValueNode; +import jdk.graal.compiler.nodes.gc.ObjectWriteBarrierNode; +import jdk.graal.compiler.nodes.memory.address.AddressNode; +import jdk.graal.compiler.nodes.spi.LIRLowerable; +import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; +import jdk.vm.ci.meta.AllocatableValue; +import jdk.vm.ci.meta.Value; + +import static jdk.graal.compiler.nodeinfo.NodeCycles.CYCLES_64; +import static jdk.graal.compiler.nodeinfo.NodeSize.SIZE_64; + +/** + * Shenandoah SATB barrier. Supports concurrent marking, by implementing the so-called + * snapshot-at-the-beginning (SATB). The barrier ensures that we see a consistent and complete + * marking bitmap after concurrent marking, that has at least all objects marked live that have been + * live at the beginning of marking (hence the name). This barrier is very similar to G1's + * pre-write-barrier. + */ +@NodeInfo(cycles = CYCLES_64, size = SIZE_64) +public final class ShenandoahSATBBarrierNode extends ObjectWriteBarrierNode implements LIRLowerable { + public static final NodeClass TYPE = NodeClass.create(ShenandoahSATBBarrierNode.class); + + public ShenandoahSATBBarrierNode(AddressNode address, ValueNode expectedObject) { + super(TYPE, address, expectedObject, true); + } + + public ValueNode getExpectedObject() { + return getValue(); + } + + @Override + public Kind getKind() { + return Kind.PRE_BARRIER; + } + + @Override + public void generate(NodeLIRBuilderTool generator) { + ValueNode expectedObject = getExpectedObject(); + if (expectedObject == null || !expectedObject.isJavaConstant() || !expectedObject.asJavaConstant().isNull()) { + AllocatableValue operand = Value.ILLEGAL; + boolean nonNull = false; + LIRGeneratorTool lirGen = generator.getLIRGeneratorTool(); + if (expectedObject != null) { + operand = lirGen.asAllocatable(generator.operand(expectedObject)); + nonNull = ((ObjectStamp) expectedObject.stamp(NodeView.DEFAULT)).nonNull(); + GraalError.guarantee(expectedObject.stamp(NodeView.DEFAULT) instanceof ObjectStamp, "expecting full size object"); + } + ShenandoahBarrierSetLIRGeneratorTool tool = (ShenandoahBarrierSetLIRGeneratorTool) generator.getLIRGeneratorTool().getBarrierSet(); + tool.emitPreWriteBarrier(lirGen, generator.operand(getAddress()), operand, nonNull); + } + } +} diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/memory/FloatingReadNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/memory/FloatingReadNode.java index 752f4a22e1cc..10590083e4e8 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/memory/FloatingReadNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/nodes/memory/FloatingReadNode.java @@ -30,9 +30,7 @@ import org.graalvm.word.LocationIdentity; -import jdk.graal.compiler.core.common.LIRKind; import jdk.graal.compiler.core.common.memory.BarrierType; -import jdk.graal.compiler.core.common.memory.MemoryExtendKind; import jdk.graal.compiler.core.common.memory.MemoryOrderMode; import jdk.graal.compiler.core.common.spi.ConstantFieldProvider; import jdk.graal.compiler.core.common.type.ObjectStamp; @@ -40,7 +38,6 @@ import jdk.graal.compiler.debug.DebugCloseable; import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.graph.NodeClass; -import jdk.graal.compiler.lir.gen.ReadBarrierSetLIRGeneratorTool; import jdk.graal.compiler.nodeinfo.NodeInfo; import jdk.graal.compiler.nodes.GraphState; import jdk.graal.compiler.nodes.GuardedValueNode; @@ -54,7 +51,6 @@ import jdk.graal.compiler.nodes.memory.address.AddressNode; import jdk.graal.compiler.nodes.spi.Canonicalizable; import jdk.graal.compiler.nodes.spi.CanonicalizerTool; -import jdk.graal.compiler.nodes.spi.NodeLIRBuilderTool; import jdk.vm.ci.meta.ResolvedJavaField; /** @@ -62,7 +58,7 @@ * relative location. This node does not null check the object. */ @NodeInfo(nameTemplate = "Read#{p#location/s}", cycles = CYCLES_2, size = SIZE_1) -public class FloatingReadNode extends FloatingAccessNode implements LIRLowerableAccess, Canonicalizable { +public class FloatingReadNode extends FloatingAccessNode implements Canonicalizable { public static final NodeClass TYPE = NodeClass.create(FloatingReadNode.class); @OptionalInput(Memory) MemoryKill lastLocationAccess; @@ -127,16 +123,6 @@ public void setLastLocationAccess(MemoryKill newlla) { lastLocationAccess = newlla; } - @Override - public void generate(NodeLIRBuilderTool gen) { - LIRKind readKind = gen.getLIRGeneratorTool().getLIRKind(stamp(NodeView.DEFAULT)); - if (getBarrierType() != BarrierType.NONE && gen.getLIRGeneratorTool().getBarrierSet() instanceof ReadBarrierSetLIRGeneratorTool barrierSet) { - gen.setResult(this, barrierSet.emitBarrieredLoad(gen.getLIRGeneratorTool(), readKind, gen.operand(address), null, MemoryOrderMode.PLAIN, getBarrierType())); - } else { - gen.setResult(this, gen.getLIRGeneratorTool().getArithmetic().emitLoad(readKind, gen.operand(address), null, MemoryOrderMode.PLAIN, MemoryExtendKind.DEFAULT)); - } - } - @Override public Node canonical(CanonicalizerTool tool) { if (!tool.canonicalizeReads()) { @@ -149,7 +135,7 @@ public Node canonical(CanonicalizerTool tool) { } if (getAddress().hasMoreThanOneUsage() && lastLocationAccess instanceof WriteNode) { WriteNode write = (WriteNode) lastLocationAccess; - if (write.getAddress() == getAddress() && write.getAccessStamp(NodeView.DEFAULT).isCompatible(getAccessStamp(NodeView.DEFAULT))) { + if (write.getAddress() == getAddress() && write.getAccessStamp(NodeView.DEFAULT).isCompatible(stamp(NodeView.DEFAULT))) { // Same memory location with no intervening write return write.value(); } @@ -244,11 +230,6 @@ public boolean verifyNode() { return super.verifyNode(); } - @Override - public Stamp getAccessStamp(NodeView view) { - return stamp(view); - } - /** * Indicate whether this load may have anti-dependencies on other stores. Anti-dependency means * that a store can overwrite the memory location which is read by this load. As a result, the diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/AddressLoweringByUsePhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/AddressLoweringByUsePhase.java index 4ef39310914f..eb43abde491e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/AddressLoweringByUsePhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/AddressLoweringByUsePhase.java @@ -37,13 +37,11 @@ import jdk.graal.compiler.nodes.ValueNode; import jdk.graal.compiler.nodes.extended.JavaReadNode; import jdk.graal.compiler.nodes.memory.AbstractWriteNode; -import jdk.graal.compiler.nodes.memory.FloatingReadNode; import jdk.graal.compiler.nodes.memory.ReadNode; import jdk.graal.compiler.nodes.memory.address.AddressNode; import jdk.graal.compiler.nodes.memory.address.OffsetAddressNode; import jdk.graal.compiler.nodes.spi.CoreProviders; import jdk.graal.compiler.nodes.util.GraphUtil; - import jdk.vm.ci.meta.JavaKind; /** @@ -85,11 +83,6 @@ protected void run(StructuredGraph graph, CoreProviders providers) { Stamp stamp = javaReadNode.stamp(NodeView.DEFAULT); address = javaReadNode.getAddress(); lowered = lowering.lower(javaReadNode, stamp, address); - } else if (node instanceof FloatingReadNode) { - FloatingReadNode floatingReadNode = (FloatingReadNode) node; - Stamp stamp = floatingReadNode.getAccessStamp(NodeView.DEFAULT); - address = floatingReadNode.getAddress(); - lowered = lowering.lower(floatingReadNode, stamp, address); } else if (node instanceof AbstractWriteNode) { AbstractWriteNode abstractWriteNode = (AbstractWriteNode) node; Stamp stamp = abstractWriteNode.value().stamp(NodeView.DEFAULT); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/FloatingReadPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/FloatingReadPhase.java index 10ead3595333..ebf655f28368 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/FloatingReadPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/FloatingReadPhase.java @@ -131,17 +131,14 @@ public EconomicMap getMap() { } public FloatingReadPhase(CanonicalizerPhase canonicalizer) { - this(false, canonicalizer); + this(false, true, canonicalizer); } /** - * @param createMemoryMapNodes a {@link MemoryMapNode} will be created for each return if this + * @param createMemoryMapNodes a {@link MemoryMapNode} will be created for each return if true + * @param createFloatingReads attempt to float {@link FloatableAccessNode} * @param canonicalizer */ - public FloatingReadPhase(boolean createMemoryMapNodes, CanonicalizerPhase canonicalizer) { - this(createMemoryMapNodes, true, canonicalizer); - } - public FloatingReadPhase(boolean createMemoryMapNodes, boolean createFloatingReads, CanonicalizerPhase canonicalizer) { super(canonicalizer); this.createMemoryMapNodes = createMemoryMapNodes; @@ -260,7 +257,9 @@ protected void run(StructuredGraph graph, CoreProviders context) { @Override public void updateGraphState(GraphState graphState) { super.updateGraphState(graphState); - graphState.setAfterStage(StageFlag.FLOATING_READS); + if (createFloatingReads) { + graphState.setAfterStage(StageFlag.FLOATING_READS); + } } @SuppressWarnings("try") diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LockEliminationPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LockEliminationPhase.java index d41eb7d05351..5bcb2b8d1c1c 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LockEliminationPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/LockEliminationPhase.java @@ -30,7 +30,6 @@ import jdk.graal.compiler.graph.Node; import jdk.graal.compiler.nodes.FixedNode; import jdk.graal.compiler.nodes.GraphState; -import jdk.graal.compiler.nodes.GraphState.StageFlag; import jdk.graal.compiler.nodes.GuardNode; import jdk.graal.compiler.nodes.PiNode; import jdk.graal.compiler.nodes.ProxyNode; @@ -56,7 +55,7 @@ public class LockEliminationPhase extends Phase { @Override public Optional notApplicableTo(GraphState graphState) { - return NotApplicable.when(graphState.isAfterStage(StageFlag.FLOATING_READS) && graphState.isBeforeStage(StageFlag.FIXED_READS), + return NotApplicable.when(graphState.allowsFloatingReads(), "This phase must not be applied while reads are floating"); } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java index 374224ea0672..1e9b49bf4f07 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/phases/common/WriteBarrierAdditionPhase.java @@ -37,11 +37,21 @@ public class WriteBarrierAdditionPhase extends BasePhase { + private final StageFlag stage; + + public WriteBarrierAdditionPhase() { + this(StageFlag.BARRIER_ADDITION); + } + + public WriteBarrierAdditionPhase(StageFlag stage) { + this.stage = stage; + } + @Override public Optional notApplicableTo(GraphState graphState) { return NotApplicable.ifAny( - NotApplicable.ifApplied(this, StageFlag.BARRIER_ADDITION, graphState), - NotApplicable.unlessRunAfter(this, StageFlag.MID_TIER_LOWERING, graphState), + NotApplicable.ifApplied(this, stage, graphState), + NotApplicable.unlessRunAfter(this, stage == StageFlag.BARRIER_ADDITION ? StageFlag.MID_TIER_LOWERING : StageFlag.LOW_TIER_LOWERING, graphState), NotApplicable.unlessRunAfter(this, StageFlag.FSA, graphState)); } @@ -49,7 +59,7 @@ public Optional notApplicableTo(GraphState graphState) { @Override protected void run(StructuredGraph graph, CoreProviders context) { BarrierSet barrierSet = context.getPlatformConfigurationProvider().getBarrierSet(); - if (barrierSet.hasWriteBarrier()) { + if (barrierSet.hasWriteBarrier() && barrierSet.shouldAddBarriersInStage(stage)) { for (FixedAccessNode n : graph.getNodes(FixedAccessNode.TYPE)) { try (DebugCloseable scope = n.graph().withNodeSourcePosition(n)) { barrierSet.addBarriers(n, context); @@ -61,7 +71,7 @@ protected void run(StructuredGraph graph, CoreProviders context) { @Override public void updateGraphState(GraphState graphState) { super.updateGraphState(graphState); - graphState.setAfterStage(StageFlag.BARRIER_ADDITION); + graphState.setAfterStage(stage); } @Override diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/BoxingSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/BoxingSnippets.java index e500806b9d8f..345c6c958e08 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/BoxingSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/BoxingSnippets.java @@ -262,7 +262,7 @@ private static LocationIdentity getCacheLocation(CoreProviders providers, JavaKi public void lower(BoxNode box, LoweringTool tool) { SnippetTemplate.Arguments args = null; - args = new SnippetTemplate.Arguments(boxSnippets.get(box.getBoxingKind()), box.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(boxSnippets.get(box.getBoxingKind()), box.graph(), tool.getLoweringStage()); args.add("value", box.getValue()); args.add("valueOfCounter", valueOfCounter); SnippetTemplate template = template(tool, box, args); @@ -273,7 +273,7 @@ public void lower(BoxNode box, LoweringTool tool) { } public void lower(UnboxNode unbox, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(unboxSnippets.get(unbox.getBoxingKind()), unbox.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(unboxSnippets.get(unbox.getBoxingKind()), unbox.graph(), tool.getLoweringStage()); args.add("value", unbox.getValue()); args.add("valueCounter", valueCounter); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IdentityHashCodeSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IdentityHashCodeSnippets.java index 6f6cc16bd8c7..6932a2658994 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IdentityHashCodeSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IdentityHashCodeSnippets.java @@ -69,7 +69,7 @@ public Templates(IdentityHashCodeSnippets receiver, OptionValues options, Provid public void lower(IdentityHashCodeNode node, LoweringTool tool) { StructuredGraph graph = node.graph(); - Arguments args = new Arguments(identityHashCodeSnippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(identityHashCodeSnippet, graph, tool.getLoweringStage()); args.add("thisObj", node.object()); SnippetTemplate template = template(tool, node, args); template.instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IsArraySnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IsArraySnippets.java index 1684c2ba2400..24355c321cf3 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IsArraySnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/IsArraySnippets.java @@ -65,10 +65,10 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac ValueNode node = replacer.instanceOf; SnippetTemplate.Arguments args; if (node instanceof ObjectIsArrayNode) { - args = new SnippetTemplate.Arguments(objectIsArraySnippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(objectIsArraySnippet, node.graph(), tool.getLoweringStage()); args.add("object", ((ObjectIsArrayNode) node).getValue()); } else if (replacer.instanceOf instanceof ClassIsArrayNode) { - args = new SnippetTemplate.Arguments(classIsArraySnippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(classIsArraySnippet, node.graph(), tool.getLoweringStage()); args.add("clazz", ((ClassIsArrayNode) node).getValue()); } else { throw GraalError.shouldNotReachHere(node + " " + replacer); // ExcludeFromJacocoGeneratedReport diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetCounterNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetCounterNode.java index 3c78d580c9a0..3fb6873211d8 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetCounterNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetCounterNode.java @@ -154,7 +154,7 @@ public static class Templates extends AbstractTemplates { public void lower(SnippetCounterNode counter, LoweringTool tool) { StructuredGraph graph = counter.graph(); - Arguments args = new Arguments(add, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(add, graph, tool.getLoweringStage()); args.add("counter", counter.getCounter()); args.add("increment", counter.getIncrement()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetSubstitutionNode.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetSubstitutionNode.java index 136b383d885c..351dbf3d7c14 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetSubstitutionNode.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetSubstitutionNode.java @@ -62,7 +62,7 @@ public void setConstantArguments(Object[] arguments) { @Override public void lower(LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, this.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, this.graph(), tool.getLoweringStage()); int arg = 0; for (; arg < arguments.size(); arg++) { args.add(snippet.getParameterName(arg), arguments.get(arg)); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetTemplate.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetTemplate.java index a3d68d1fd70b..185677ce1d56 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetTemplate.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/SnippetTemplate.java @@ -45,6 +45,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; @@ -630,9 +631,13 @@ public static final class Arguments implements Formattable { protected int nextParamIdx; - public Arguments(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { + public Arguments(SnippetInfo info, StructuredGraph graph, LoweringTool.LoweringStage loweringStage) { + this(info, graph.getGuardsStage(), graph.getGraphState().allowsFloatingReads(), loweringStage); + } + + public Arguments(SnippetInfo info, GuardsStage guardsStage, boolean allowsFloatingReads, LoweringTool.LoweringStage loweringStage) { this.info = info; - this.cacheKey = new CacheKey(info, guardsStage, loweringStage); + this.cacheKey = new CacheKey(info, guardsStage, allowsFloatingReads, loweringStage); this.values = new Object[info.getParameterCount()]; this.constStamps = new Stamp[info.getParameterCount()]; this.cacheable = true; @@ -803,16 +808,18 @@ public static class CacheKey { private final ResolvedJavaMethod method; private final Object[] values; private final GuardsStage guardsStage; + private final boolean allowsFloatingReads; private final LoweringTool.LoweringStage loweringStage; private int hash; private final Snippet.SnippetType type; - protected CacheKey(SnippetInfo info, GuardsStage guardsStage, LoweringTool.LoweringStage loweringStage) { + protected CacheKey(SnippetInfo info, GuardsStage guardsStage, boolean allowsFloatingReads, LoweringTool.LoweringStage loweringStage) { this.method = info.method; this.guardsStage = guardsStage; + this.allowsFloatingReads = allowsFloatingReads; this.loweringStage = loweringStage; this.values = new Object[info.getParameterCount()]; - this.hash = info.method.hashCode() + 31 * guardsStage.ordinal(); + this.hash = Objects.hash(info.method, allowsFloatingReads, loweringStage); this.type = info.type; } @@ -830,7 +837,7 @@ public boolean equals(Object obj) { if (!method.equals(other.method)) { return false; } - if (guardsStage != other.guardsStage || loweringStage != other.loweringStage) { + if (guardsStage != other.guardsStage || loweringStage != other.loweringStage || allowsFloatingReads != other.allowsFloatingReads) { return false; } if (type != other.type) { @@ -1240,6 +1247,7 @@ protected SnippetTemplate(OptionValues options, boolean needsPEA = false; boolean needsCE = false; LoweringTool.LoweringStage loweringStage = args.cacheKey.loweringStage; + boolean allowsFloatingReads = args.cacheKey.allowsFloatingReads; for (Node n : snippetCopy.getNodes()) { if (!needsPEA && (n instanceof AbstractNewObjectNode || n instanceof AbstractBoxingNode)) { needsPEA = true; @@ -1273,8 +1281,8 @@ protected SnippetTemplate(OptionValues options, if (loweringStage != LoweringTool.StandardLoweringStage.HIGH_TIER) { // (3) assert !guardsStage.allowsFloatingGuards() : guardsStage; - // only create memory map nodes if we need the memory graph - new FloatingReadPhase(true, false, canonicalizer).apply(snippetCopy, providers); + // Create memory map nodes + new FloatingReadPhase(true, allowsFloatingReads, canonicalizer).apply(snippetCopy, providers); if (!snippetCopy.getGraphState().isExplicitExceptionsNoDeopt()) { new GuardLoweringPhase().apply(snippetCopy, providers); @@ -1957,9 +1965,7 @@ public Collection getLocations() { private void rewireMemoryGraph(ValueNode replacee, UnmodifiableEconomicMap duplicates) { verifyWithExceptionNode(replacee); - final StructuredGraph replaceeGraph = replacee.graph(); - final boolean hasMemoryGraph = replaceeGraph.isAfterStage(StageFlag.FLOATING_READS) && !replaceeGraph.isAfterStage(StageFlag.FIXED_READS); - if (!hasMemoryGraph) { + if (!replacee.graph().getGraphState().allowsFloatingReads()) { return; } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64IntegerArithmeticSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64IntegerArithmeticSnippets.java index aef616c83e1b..1b9f6bc95eae 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64IntegerArithmeticSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/aarch64/AArch64IntegerArithmeticSnippets.java @@ -111,7 +111,7 @@ public void lower(IntegerDivRemNode node, LoweringTool tool) { throw GraalError.shouldNotReachHereUnexpectedValue(node); // ExcludeFromJacocoGeneratedReport } StructuredGraph graph = node.graph(); - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, graph, tool.getLoweringStage()); args.add("x", node.getX()); args.add("y", node.getY()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/arraycopy/ArrayCopySnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/arraycopy/ArrayCopySnippets.java index 6b9b8fa5b0b9..491d5f489edf 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/arraycopy/ArrayCopySnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/arraycopy/ArrayCopySnippets.java @@ -715,7 +715,7 @@ public void lower(ArrayCopyNode arraycopy, boolean mayExpandThisArraycopy, Lower } // create the snippet - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippetInfo, arraycopy.graph(), tool.getLoweringStage()); args.add("src", arraycopy.getSource()); args.add("srcPos", arraycopy.getSourcePosition()); args.add("dest", arraycopy.getDestination()); @@ -751,7 +751,7 @@ public void lower(ArrayCopyWithDelayedLoweringNode arraycopy, LoweringTool tool) return; } - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(getSnippet(arraycopy.getSnippet()), arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(getSnippet(arraycopy.getSnippet()), arraycopy.graph(), tool.getLoweringStage()); args.add("src", arraycopy.getSource()); args.add("srcPos", arraycopy.getSourcePosition()); args.add("dest", arraycopy.getDestination()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/G1WriteBarrierSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/G1WriteBarrierSnippets.java index e0283a1fd683..40fc23c15993 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/G1WriteBarrierSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/G1WriteBarrierSnippets.java @@ -452,7 +452,7 @@ public G1WriteBarrierLowerer(SnippetCounter.Group.Factory factory) { } public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.SnippetInfo snippet, G1PreWriteBarrierNode barrier, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); AddressNode address = barrier.getAddress(); args.add("address", address); if (address instanceof OffsetAddressNode) { @@ -475,7 +475,7 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S } public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.SnippetInfo snippet, G1ReferentFieldReadBarrierNode barrier, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); // This is expected to be lowered before address lowering OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress(); args.add("address", address); @@ -499,7 +499,7 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S return; } - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); AddressNode address = barrier.getAddress(); args.add("address", address); if (address instanceof OffsetAddressNode) { @@ -523,7 +523,7 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S } public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.SnippetInfo snippet, G1ArrayRangePreWriteBarrierNode barrier, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); args.add("address", barrier.getAddress()); args.add("length", barrier.getLengthAsLong()); args.add("elementStride", barrier.getElementStride()); @@ -532,7 +532,7 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S } public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.SnippetInfo snippet, G1ArrayRangePostWriteBarrierNode barrier, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); args.add("address", barrier.getAddress()); args.add("length", barrier.getLengthAsLong()); args.add("elementStride", barrier.getElementStride()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/SerialWriteBarrierSnippets.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/SerialWriteBarrierSnippets.java index a2d21764d8ed..f8852c8a08b9 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/SerialWriteBarrierSnippets.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/replacements/gc/SerialWriteBarrierSnippets.java @@ -118,10 +118,10 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S LoweringTool tool) { SnippetTemplate.Arguments args; if (barrier.usePrecise()) { - args = new SnippetTemplate.Arguments(preciseSnippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(preciseSnippet, barrier.graph(), tool.getLoweringStage()); args.add("address", barrier.getAddress()); } else { - args = new SnippetTemplate.Arguments(impreciseSnippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(impreciseSnippet, barrier.graph(), tool.getLoweringStage()); OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress(); args.add("object", address.getBase()); } @@ -132,7 +132,7 @@ public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.S } public void lower(SnippetTemplate.AbstractTemplates templates, SnippetTemplate.SnippetInfo snippet, SerialArrayRangeWriteBarrierNode barrier, LoweringTool tool) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(snippet, barrier.graph(), tool.getLoweringStage()); args.add("address", barrier.getAddress()); args.add("length", barrier.getLengthAsLong()); args.add("elementStride", barrier.getElementStride()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleSafepointLoweringSnippet.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleSafepointLoweringSnippet.java index ebfa1b269aa6..9719074d47cf 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleSafepointLoweringSnippet.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/HotSpotTruffleSafepointLoweringSnippet.java @@ -115,7 +115,7 @@ static class Templates extends AbstractTemplates { public void lower(TruffleSafepointNode node, LoweringTool tool, ResolvedJavaMethod javaMethod) { StructuredGraph graph = node.graph(); - Arguments args = new Arguments(pollSnippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(pollSnippet, graph, tool.getLoweringStage()); ValueNode method = ConstantNode.forConstant(tool.getStampProvider().createMethodStamp(), javaMethod.getEncoding(), tool.getMetaAccess(), graph); args.add("method", method); args.add("node", node.location()); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java index e6b31813498d..d2a9fa28358e 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/TruffleEntryPointDecorator.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.truffle.hotspot; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; @@ -65,5 +66,9 @@ public void initialize(CoreProviders providers, LIRGenerationResult lirGenRes) { ForeignCallLinkage callTarget = providers.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); lirGenRes.getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + ForeignCallLinkage callTarget = providers.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + lirGenRes.getFrameMapBuilder().callsMethod(callTarget.getOutgoingCallingConvention()); + } } } diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java index cedf561593f1..bfaaf8e8e596 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/aarch64/AArch64TruffleCallBoundaryInstrumentationFactory.java @@ -24,6 +24,7 @@ */ package jdk.graal.compiler.truffle.hotspot.aarch64; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import static jdk.vm.ci.hotspot.HotSpotCallingConventionType.JavaCall; import static jdk.vm.ci.meta.JavaKind.Object; @@ -38,10 +39,12 @@ import jdk.graal.compiler.hotspot.HotSpotGraalRuntime; import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotBackend; import jdk.graal.compiler.hotspot.aarch64.AArch64HotSpotMove; +import jdk.graal.compiler.hotspot.aarch64.shenandoah.AArch64HotSpotShenandoahLoadRefBarrierOp; import jdk.graal.compiler.hotspot.aarch64.z.AArch64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.meta.HotSpotRegistersProvider; import jdk.graal.compiler.lir.asm.CompilationResultBuilder; import jdk.graal.compiler.lir.asm.EntryPointDecorator; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; import jdk.graal.compiler.serviceprovider.ServiceProvider; import jdk.graal.compiler.truffle.TruffleCompilerConfiguration; import jdk.graal.compiler.truffle.hotspot.TruffleCallBoundaryInstrumentationFactory; @@ -56,7 +59,7 @@ public EntryPointDecorator create(TruffleCompilerConfiguration compilerConfig, G return new TruffleEntryPointDecorator(compilerConfig, config, registers) { @Override public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetup) { - if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z)) { + if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z || config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah)) { // The Z load barrier must be performed after the nmethod entry barrier which is // part of the frame setup. The other GCs don't have a read barrier so it's // safe to do this dispatch before the frame is set up. @@ -72,9 +75,16 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu Label doProlog = new Label(); if (config.useCompressedOops) { CompressEncoding encoding = config.getOopEncoding(); - masm.ldr(32, spillRegister, AArch64Address.createImmediateAddress(32, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset)); + AArch64Address address = AArch64Address.createImmediateAddress(32, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset); + masm.ldr(32, spillRegister, address); Register base = encoding.hasBase() ? registers.getHeapBaseRegister() : null; AArch64HotSpotMove.UncompressPointer.emitUncompressCode(masm, spillRegister, spillRegister, base, encoding, true); + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + AArch64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } else { AArch64Address address = AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, thisRegister, installedCodeOffset); masm.ldr(64, spillRegister, address); @@ -82,6 +92,12 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); AArch64HotSpotZBarrierSetLIRGenerator.emitLoadBarrier(crb, masm, config, spillRegister, callTarget, address, null, false, false); } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + AArch64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, spillRegister, spillRegister, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + } } masm.ldr(64, spillRegister, AArch64Address.createImmediateAddress(64, AArch64Address.AddressingMode.IMMEDIATE_UNSIGNED_SCALED, spillRegister, entryPointOffset)); masm.cbz(64, spillRegister, doProlog); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java index e45fcbd170ee..f816801470e1 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/truffle/hotspot/amd64/AMD64TruffleCallBoundaryInstrumentationFactory.java @@ -24,23 +24,29 @@ */ package jdk.graal.compiler.truffle.hotspot.amd64; +import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.SHENANDOAH_LOAD_BARRIER; import static jdk.graal.compiler.hotspot.meta.HotSpotHostForeignCallsProvider.Z_LOAD_BARRIER; import static jdk.vm.ci.hotspot.HotSpotCallingConventionType.JavaCall; +import java.util.List; + import jdk.graal.compiler.asm.Label; import jdk.graal.compiler.asm.amd64.AMD64Address; import jdk.graal.compiler.asm.amd64.AMD64Assembler.ConditionFlag; import jdk.graal.compiler.asm.amd64.AMD64MacroAssembler; import jdk.graal.compiler.core.common.CompressEncoding; import jdk.graal.compiler.core.common.spi.ForeignCallLinkage; +import jdk.graal.compiler.debug.GraalError; import jdk.graal.compiler.hotspot.GraalHotSpotVMConfig; import jdk.graal.compiler.hotspot.HotSpotGraalRuntime; import jdk.graal.compiler.hotspot.amd64.AMD64HotSpotBackend; +import jdk.graal.compiler.hotspot.amd64.shenandoah.AMD64HotSpotShenandoahLoadRefBarrierOp; import jdk.graal.compiler.hotspot.amd64.z.AMD64HotSpotZBarrierSetLIRGenerator; import jdk.graal.compiler.hotspot.meta.HotSpotRegistersProvider; import jdk.graal.compiler.lir.amd64.AMD64Move; import jdk.graal.compiler.lir.asm.CompilationResultBuilder; import jdk.graal.compiler.lir.asm.EntryPointDecorator; +import jdk.graal.compiler.nodes.gc.shenandoah.ShenandoahLoadRefBarrierNode; import jdk.graal.compiler.serviceprovider.ServiceProvider; import jdk.graal.compiler.truffle.TruffleCompilerConfiguration; import jdk.graal.compiler.truffle.hotspot.TruffleCallBoundaryInstrumentationFactory; @@ -57,14 +63,15 @@ public EntryPointDecorator create(TruffleCompilerConfiguration compilerConfig, G return new TruffleEntryPointDecorator(compilerConfig, config, registers) { @Override public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetup) { - if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z)) { + if (beforeFrameSetup == (config.gc == HotSpotGraalRuntime.HotSpotGC.Z || config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah)) { // The Z load barrier must be performed after the nmethod entry barrier which is // part of the frame setup. The other GCs don't have a read barrier so it's // safe to do this dispatch before the frame is set up. return; } AMD64MacroAssembler masm = (AMD64MacroAssembler) crb.asm; - Register thisRegister = crb.getCodeCache().getRegisterConfig().getCallingConventionRegisters(JavaCall, JavaKind.Object).get(0); + List callRegisters = crb.getCodeCache().getRegisterConfig().getCallingConventionRegisters(JavaCall, JavaKind.Object); + Register thisRegister = callRegisters.get(0); Register spillRegister = AMD64.r10; Label doProlog = new Label(); int pos = masm.position(); @@ -74,7 +81,7 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu // First instruction must be at least 5 bytes long to be safe for // patching masm.movl(spillRegister, address, beforeFrameSetup); - assert masm.position() - pos >= AMD64HotSpotBackend.PATCHED_VERIFIED_ENTRY_POINT_INSTRUCTION_SIZE : masm.position() + "-" + pos; + assert !beforeFrameSetup || masm.position() - pos >= AMD64HotSpotBackend.PATCHED_VERIFIED_ENTRY_POINT_INSTRUCTION_SIZE : masm.position() + "-" + pos; CompressEncoding encoding = config.getOopEncoding(); Register heapBaseRegister = AMD64Move.UncompressPointerOp.hasBase(encoding) ? registers.getHeapBaseRegister() : Register.None; AMD64Move.UncompressPointerOp.emitUncompressCode(masm, spillRegister, encoding.getShift(), heapBaseRegister, true); @@ -83,11 +90,24 @@ public void emitEntryPoint(CompilationResultBuilder crb, boolean beforeFrameSetu // patching masm.movq(spillRegister, address, beforeFrameSetup); assert masm.position() - pos >= AMD64HotSpotBackend.PATCHED_VERIFIED_ENTRY_POINT_INSTRUCTION_SIZE : masm.position() + "-" + pos; - if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { - ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); - AMD64HotSpotZBarrierSetLIRGenerator.emitLoadBarrier(crb, masm, spillRegister, callTarget, address, null, - false); - } + } + if (config.gc == HotSpotGraalRuntime.HotSpotGC.Z) { + GraalError.guarantee(!config.useCompressedOops, "only uncompressed oops"); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(Z_LOAD_BARRIER); + AMD64HotSpotZBarrierSetLIRGenerator.emitLoadBarrier(crb, masm, spillRegister, callTarget, address, null, + false); + } else if (config.gc == HotSpotGraalRuntime.HotSpotGC.Shenandoah) { + Register thread = registers.getThreadRegister(); + ForeignCallLinkage callTarget = crb.getForeignCalls().lookupForeignCall(SHENANDOAH_LOAD_BARRIER); + Register tmp1 = AMD64.r11; + Register tmp2 = AMD64.r13; + Register objectRegister = AMD64.r14; + assert !callRegisters.contains(tmp1); + assert !callRegisters.contains(tmp2); + assert !callRegisters.contains(objectRegister); + AMD64HotSpotShenandoahLoadRefBarrierOp.emitCode(config, crb, masm, null, thread, objectRegister, spillRegister, tmp1, tmp2, address, callTarget, + ShenandoahLoadRefBarrierNode.ReferenceStrength.STRONG, false); + spillRegister = objectRegister; } masm.movq(spillRegister, new AMD64Address(spillRegister, entryPointOffset)); masm.testqAndJcc(spillRegister, spillRegister, ConditionFlag.Equal, doProlog, true); diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/ReadEliminationPhase.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/ReadEliminationPhase.java index 2b655e140440..0f4508e84873 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/ReadEliminationPhase.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/virtual/phases/ea/ReadEliminationPhase.java @@ -28,8 +28,9 @@ import java.util.Optional; +import org.graalvm.word.LocationIdentity; + import jdk.graal.compiler.nodes.GraphState; -import jdk.graal.compiler.nodes.GraphState.StageFlag; import jdk.graal.compiler.nodes.StructuredGraph; import jdk.graal.compiler.nodes.StructuredGraph.ScheduleResult; import jdk.graal.compiler.nodes.cfg.ControlFlowGraph; @@ -37,7 +38,6 @@ import jdk.graal.compiler.options.OptionValues; import jdk.graal.compiler.phases.common.CanonicalizerPhase; import jdk.graal.compiler.phases.graph.ReentrantBlockIterator; -import org.graalvm.word.LocationIdentity; /** * This phase performs read and (simple) write elimination on a graph. It operates on multiple @@ -82,7 +82,7 @@ public ReadEliminationPhase(CanonicalizerPhase canonicalizer, boolean considerGu public Optional notApplicableTo(GraphState graphState) { return NotApplicable.ifAny( super.notApplicableTo(graphState), - NotApplicable.when(graphState.isAfterStage(StageFlag.FLOATING_READS) && graphState.isBeforeStage(StageFlag.FIXED_READS), + NotApplicable.when(graphState.allowsFloatingReads(), "This phase must not be applied while reads are floating")); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index 6ba052cbadc8..a1322177a135 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -243,7 +243,7 @@ private class PostWriteBarrierLowering implements NodeLoweringProvider { @Override public void lower(EnsureClassInitializedNode node, LoweringTool tool) { - Arguments args = new Arguments(ensureClassIsInitialized, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(ensureClassIsInitialized, node.graph(), tool.getLoweringStage()); args.add("hub", node.getHub()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java index fb31d6ab8854..ae7aeeac0f88 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/jdk/SubstrateObjectCloneSnippets.java @@ -248,7 +248,7 @@ public void lower(SubstrateObjectCloneNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(doClone, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(doClone, node.graph(), tool.getLoweringStage()); args.add("thisObj", node.getObject()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ArithmeticSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ArithmeticSnippets.java index 382290b226ce..06c686f2d62f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ArithmeticSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ArithmeticSnippets.java @@ -259,7 +259,7 @@ public void lower(IntegerDivRemNode node, LoweringTool tool) { } else { throw shouldNotReachHereUnexpectedInput(node); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, node.graph(), tool.getLoweringStage()); args.add("x", node.getX()); args.add("y", node.getY()); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java index ed7da2af7dcb..552b1a567dc2 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java @@ -940,21 +940,21 @@ public void lower(CEntryPointEnterNode node, LoweringTool tool) { Arguments args; switch (node.getEnterAction()) { case CreateIsolate: - args = new Arguments(createIsolate, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(createIsolate, node.graph(), tool.getLoweringStage()); args.add("parameters", node.getParameter()); break; case AttachThread: - args = new Arguments(attachThread, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(attachThread, node.graph(), tool.getLoweringStage()); args.add("isolate", node.getParameter()); args.add("startedByIsolate", node.getStartedByIsolate()); args.add("ensureJavaThread", node.getEnsureJavaThread()); break; case EnterByIsolate: - args = new Arguments(enterByIsolate, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(enterByIsolate, node.graph(), tool.getLoweringStage()); args.add("isolate", node.getParameter()); break; case Enter: - args = new Arguments(enter, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(enter, node.graph(), tool.getLoweringStage()); assert node.getParameter() != null; args.add("thread", node.getParameter()); break; @@ -976,16 +976,16 @@ public void lower(CEntryPointLeaveNode node, LoweringTool tool) { Arguments args; switch (node.getLeaveAction()) { case Leave: - args = new Arguments(returnFromJavaToC, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(returnFromJavaToC, node.graph(), tool.getLoweringStage()); break; case DetachThread: - args = new Arguments(detachCurrentThread, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(detachCurrentThread, node.graph(), tool.getLoweringStage()); break; case TearDownIsolate: - args = new Arguments(tearDownIsolate, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(tearDownIsolate, node.graph(), tool.getLoweringStage()); break; case ExceptionAbort: - args = new Arguments(reportException, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(reportException, node.graph(), tool.getLoweringStage()); args.add("exception", node.getException()); break; default: @@ -1005,11 +1005,11 @@ public void lower(CEntryPointUtilityNode node, LoweringTool tool) { Arguments args; switch (node.getUtilityAction()) { case IsAttached: - args = new Arguments(isAttached, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(isAttached, node.graph(), tool.getLoweringStage()); args.add("isolate", node.getParameter0()); break; case FailFatally: - args = new Arguments(failFatally, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(failFatally, node.graph(), tool.getLoweringStage()); args.add("code", node.getParameter0()); args.add("message", node.getParameter1()); break; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CFunctionSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CFunctionSnippets.java index 9ac3c69c6076..082a4b2fe797 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CFunctionSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CFunctionSnippets.java @@ -178,7 +178,7 @@ public void lower(CFunctionPrologueNode node, LoweringTool tool) { public SnippetTemplate get() { int newThreadStatus = node.getNewThreadStatus(); assert StatusSupport.isValidStatus(newThreadStatus); - Arguments args = new Arguments(prologue, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(prologue, node.graph(), tool.getLoweringStage()); args.add("newThreadStatus", newThreadStatus); SnippetTemplate template = template(tool, node, args); return template; @@ -209,7 +209,7 @@ public void lower(CFunctionEpilogueNode node, LoweringTool tool) { public SnippetTemplate get() { int oldThreadStatus = node.getOldThreadStatus(); assert StatusSupport.isValidStatus(oldThreadStatus); - Arguments args = new Arguments(epilogue, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(epilogue, node.graph(), tool.getLoweringStage()); args.add("oldThreadStatus", oldThreadStatus); SnippetTemplate template = template(tool, node, args); return template; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptHostedSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptHostedSnippets.java index b22fe9aaa956..2fc0ea94de56 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptHostedSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptHostedSnippets.java @@ -212,7 +212,7 @@ public void lower(DeoptimizeNode node, LoweringTool tool) { } StructuredGraph graph = node.graph(); - Arguments args = new Arguments(deopt, graph.getGuardsStage(), loweringStage); + Arguments args = new Arguments(deopt, graph, loweringStage); args.add("reason", node.getReason()); args.add("mustNotAllocate", mustNotAllocate(graph.method())); args.add("message", message); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTestSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTestSnippets.java index 1d87b42133e0..0bd9edd13b98 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTestSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/DeoptTestSnippets.java @@ -69,7 +69,7 @@ public void lower(DeoptTestNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(deoptTest, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(deoptTest, node.graph(), tool.getLoweringStage()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ExceptionSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ExceptionSnippets.java index 43d00a51b1bd..21bd1aa2dfe3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ExceptionSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/ExceptionSnippets.java @@ -99,7 +99,7 @@ public void lower(UnwindNode node, LoweringTool tool) { */ return; } - Arguments args = new Arguments(unwind, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(unwind, node.graph(), tool.getLoweringStage()); args.add("exception", node.exception()); args.add("fromMethodWithCalleeSavedRegisters", ((SharedMethod) node.graph().method()).hasCalleeSavedRegisters()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldDispatchTableSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldDispatchTableSnippets.java index 4573939dc2db..d919dce20c3c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldDispatchTableSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldDispatchTableSnippets.java @@ -134,7 +134,7 @@ public void lower(LoadOpenTypeWorldDispatchTableStartingOffset node, LoweringToo * an interface dispatch table. */ if (target.getDeclaringClass().isInterface()) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(loadITableStartingOffset, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(loadITableStartingOffset, node.graph(), tool.getLoweringStage()); args.add("hub", node.getHub()); args.add("interfaceTypeID", ((SharedType) target.getDeclaringClass()).getTypeID()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); @@ -148,7 +148,7 @@ public void lower(LoadOpenTypeWorldDispatchTableStartingOffset node, LoweringToo /* * Otherwise we must search on the interfaceID */ - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(loadDispatchTableStartingOffset, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(loadDispatchTableStartingOffset, node.graph(), tool.getLoweringStage()); args.add("hub", node.getHub()); args.add("interfaceTypeID", node.getInterfaceTypeID()); args.add("vtableStartingOffset", vtableStartingOffset); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldSnippets.java index 4fde432c4f67..08a8ffe16b4e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/OpenTypeWorldSnippets.java @@ -244,7 +244,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac SnippetTemplate.Arguments args; if (typeReference.isExact()) { - args = new SnippetTemplate.Arguments(typeEquality, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(typeEquality, node.graph(), tool.getLoweringStage()); args.add("object", node.getValue()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -258,7 +258,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac protected SnippetTemplate.Arguments makeArgumentsForInexactType(InstanceOfUsageReplacer replacer, LoweringTool tool, InstanceOfNode node, SharedType type, DynamicHub hub) { assert type.getSingleImplementor() == null : "Canonicalization of InstanceOfNode produces exact type for single implementor"; - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOf, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOf, node.graph(), tool.getLoweringStage()); args.add("object", node.getValue()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -288,7 +288,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac InstanceOfDynamicNode node = (InstanceOfDynamicNode) replacer.instanceOf; if (node.isExact()) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(typeEquality, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(typeEquality, node.graph(), tool.getLoweringStage()); args.add("object", node.getObject()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -297,7 +297,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac return args; } else { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOfDynamic, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOfDynamic, node.graph(), tool.getLoweringStage()); args.add("type", node.getMirrorOrHub()); args.add("object", node.getObject()); args.add("trueValue", replacer.trueValue); @@ -326,7 +326,7 @@ public void lower(FloatingNode node, LoweringTool tool) { protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replacer, LoweringTool tool) { ClassIsAssignableFromNode node = (ClassIsAssignableFromNode) replacer.instanceOf; - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(assignableTypeCheck, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(assignableTypeCheck, node.graph(), tool.getLoweringStage()); args.add("type", node.getThisClass()); args.add("checkedHub", node.getOtherClass()); args.add("trueValue", replacer.trueValue); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java index 7922e03f0a9b..00910dca027f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SafepointSnippets.java @@ -101,7 +101,7 @@ public void lower(SafepointNode node, LoweringTool tool) { /* Basic sanity check to catch errors during safepoint insertion. */ throw GraalError.shouldNotReachHere("Must not insert safepoints in Uninterruptible code: " + node.stateBefore().toString(Verbosity.Debugger)); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(safepoint, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(safepoint, node.graph(), tool.getLoweringStage()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java index ee9ceb6e2974..959be797c339 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/StackOverflowCheckImpl.java @@ -514,7 +514,7 @@ public void lower(StackOverflowCheckNode node, LoweringTool tool) { long deoptFrameSize = StackOverflowCheckImpl.computeDeoptFrameSize(graph); - Arguments args = new Arguments(stackOverflowCheck, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(stackOverflowCheck, graph, tool.getLoweringStage()); args.add("mustNotAllocate", mustNotAllocatePredicate != null && mustNotAllocatePredicate.test(graph.method())); args.add("hasDeoptFrameSize", deoptFrameSize > 0); args.add("deoptFrameSize", deoptFrameSize); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java index f5c51c40e4ba..40f0bfa8c694 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java @@ -802,11 +802,11 @@ public void lower(FixedNode node, LoweringTool tool) { ValueNode objectHeaderConstant = snippetReflection.forTLABObjectHeader(hub, graph); if (objectHeaderConstant != null) { - args = new Arguments(allocateInstanceConstantHeader, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(allocateInstanceConstantHeader, graph, tool.getLoweringStage()); args.add("objectHeader", objectHeaderConstant); } else { ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(hub), tool.getMetaAccess(), graph); - args = new Arguments(allocateInstance, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(allocateInstance, graph, tool.getLoweringStage()); args.add("hub", hubConstant); } @@ -840,7 +840,7 @@ public void lower(SubstrateNewHybridInstanceNode node, LoweringTool tool) { ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(hub), tool.getMetaAccess(), graph); - Arguments args = new Arguments(allocateArray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArray, graph, tool.getLoweringStage()); args.add("hub", hubConstant); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); args.add("forceSlowPath", shouldForceSlowPath(graph)); @@ -875,7 +875,7 @@ public void lower(NewStoredContinuationNode node, LoweringTool tool) { ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(hub), tool.getMetaAccess(), graph); - Arguments args = new Arguments(allocateStoredContinuation, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateStoredContinuation, graph, tool.getLoweringStage()); args.add("hub", hubConstant); args.add("length", length.isAlive() ? length : graph.addOrUniqueWithInputs(length)); args.add("forceSlowPath", shouldForceSlowPath(graph)); @@ -926,11 +926,11 @@ public void lower(FixedNode node, LoweringTool tool) { ValueNode objectHeaderConstant = snippetReflection.forTLABObjectHeader(hub, graph); if (objectHeaderConstant != null) { - args = new Arguments(allocateArrayConstantHeader, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(allocateArrayConstantHeader, graph, tool.getLoweringStage()); args.add("objectHeader", objectHeaderConstant); } else { ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(hub), tool.getMetaAccess(), graph); - args = new Arguments(allocateArray, graph.getGuardsStage(), tool.getLoweringStage()); + args = new Arguments(allocateArray, graph, tool.getLoweringStage()); args.add("hub", hubConstant); } @@ -967,7 +967,7 @@ public void lower(NewMultiArrayNode node, LoweringTool tool) { SharedType type = (SharedType) node.type(); ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(type.getHub()), tool.getMetaAccess(), graph); - Arguments args = new Arguments(newmultiarray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(newmultiarray, graph, tool.getLoweringStage()); args.add("hub", hubConstant); args.add("rank", rank); args.add("withException", false); @@ -994,7 +994,7 @@ public void lower(NewMultiArrayWithExceptionNode node, LoweringTool tool) { SharedType type = (SharedType) node.type(); ConstantNode hubConstant = ConstantNode.forConstant(snippetReflection.forObject(type.getHub()), tool.getMetaAccess(), graph); - Arguments args = new Arguments(newmultiarray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(newmultiarray, graph, tool.getLoweringStage()); args.add("hub", hubConstant); args.add("rank", rank); args.add("withException", true); @@ -1012,7 +1012,7 @@ public void lower(DynamicNewInstanceNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(allocateInstanceDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstanceDynamic, graph, tool.getLoweringStage()); args.add("hub", node.getInstanceType()); args.add("forceSlowPath", shouldForceSlowPath(graph)); args.add("fillContents", FillContent.fromBoolean(node.fillContents())); @@ -1034,7 +1034,7 @@ public void lower(DynamicNewInstanceWithExceptionNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(allocateInstanceDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateInstanceDynamic, graph, tool.getLoweringStage()); args.add("hub", node.getInstanceType()); args.add("forceSlowPath", shouldForceSlowPath(graph)); args.add("fillContents", FillContent.fromBoolean(true)); @@ -1056,7 +1056,7 @@ public void lower(DynamicNewArrayNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArrayDynamic, graph, tool.getLoweringStage()); args.add("elementType", node.getElementType()); args.add("length", node.length()); args.add("forceSlowPath", shouldForceSlowPath(graph)); @@ -1079,7 +1079,7 @@ public void lower(DynamicNewArrayWithExceptionNode node, LoweringTool tool) { return; } - Arguments args = new Arguments(allocateArrayDynamic, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocateArrayDynamic, graph, tool.getLoweringStage()); args.add("elementType", node.getElementType()); args.add("length", node.length()); args.add("forceSlowPath", shouldForceSlowPath(graph)); @@ -1099,7 +1099,7 @@ private final class ValidateNewInstanceClassLowering implements NodeLoweringProv public void lower(ValidateNewInstanceClassNode node, LoweringTool tool) { StructuredGraph graph = node.graph(); - Arguments args = new Arguments(validateNewInstanceClass, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(validateNewInstanceClass, graph, tool.getLoweringStage()); args.add("hub", node.getInstanceType()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); @@ -1118,7 +1118,7 @@ public void lower(NewPodInstanceNode node, LoweringTool tool) { tool.getConstantReflection().asJavaType(node.getHub().asConstant()).equals(node.getKnownInstanceType())); assert node.fillContents() : "fillContents must be true for hybrid allocations"; - Arguments args = new Arguments(allocatePod, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(allocatePod, graph, tool.getLoweringStage()); args.add("hub", node.getHub()); args.add("arrayLength", node.getArrayLength()); args.add("forceSlowPath", shouldForceSlowPath(graph)); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/TypeSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/TypeSnippets.java index bd870bc398f0..37ecda48d53e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/TypeSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/TypeSnippets.java @@ -203,7 +203,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac SnippetTemplate.Arguments args; if (typeReference.isExact()) { - args = new SnippetTemplate.Arguments(typeEquality, node.graph().getGuardsStage(), tool.getLoweringStage()); + args = new SnippetTemplate.Arguments(typeEquality, node.graph(), tool.getLoweringStage()); args.add("object", node.getValue()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -217,7 +217,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac protected SnippetTemplate.Arguments makeArgumentsForInexactType(InstanceOfUsageReplacer replacer, LoweringTool tool, InstanceOfNode node, SharedType type, DynamicHub hub) { assert type.getSingleImplementor() == null : "Canonicalization of InstanceOfNode produces exact type for single implementor"; - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOf, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOf, node.graph(), tool.getLoweringStage()); args.add("object", node.getValue()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -249,7 +249,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac InstanceOfDynamicNode node = (InstanceOfDynamicNode) replacer.instanceOf; if (node.isExact()) { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(typeEquality, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(typeEquality, node.graph(), tool.getLoweringStage()); args.add("object", node.getObject()); args.add("trueValue", replacer.trueValue); args.add("falseValue", replacer.falseValue); @@ -258,7 +258,7 @@ protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replac return args; } else { - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOfDynamic, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(instanceOfDynamic, node.graph(), tool.getLoweringStage()); args.add("type", node.getMirrorOrHub()); args.add("object", node.getObject()); args.add("trueValue", replacer.trueValue); @@ -288,7 +288,7 @@ public void lower(FloatingNode node, LoweringTool tool) { protected SnippetTemplate.Arguments makeArguments(InstanceOfUsageReplacer replacer, LoweringTool tool) { ClassIsAssignableFromNode node = (ClassIsAssignableFromNode) replacer.instanceOf; - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(assignableTypeCheck, node.graph().getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(assignableTypeCheck, node.graph(), tool.getLoweringStage()); args.add("type", node.getThisClass()); args.add("checkedHub", node.getOtherClass()); args.add("trueValue", replacer.trueValue); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/AArch64ArithmeticSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/AArch64ArithmeticSnippets.java index c7c5ac1de54a..de14bcd904e8 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/AArch64ArithmeticSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/AArch64ArithmeticSnippets.java @@ -295,7 +295,7 @@ public void lower(RemNode node, LoweringTool tool) { assert kind == JavaKind.Float || kind == JavaKind.Double; SnippetTemplate.SnippetInfo snippet = kind == JavaKind.Float ? frem : drem; StructuredGraph graph = node.graph(); - Arguments args = new Arguments(snippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, graph, tool.getLoweringStage()); args.add("x", node.getX()); args.add("y", node.getY()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, tool, args); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/PosixAArch64VaListSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/PosixAArch64VaListSnippets.java index b9fb904a9049..6ca7f92d2dfe 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/PosixAArch64VaListSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/aarch64/PosixAArch64VaListSnippets.java @@ -177,7 +177,7 @@ private PosixAArch64VaListSnippets(OptionValues options, Providers providers, Ma protected class VaListInitializationSnippetsLowering implements NodeLoweringProvider { @Override public void lower(VaListInitializationNode node, LoweringTool tool) { - Arguments args = new Arguments(vaListInitialization, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(vaListInitialization, node.graph(), tool.getLoweringStage()); args.add("vaList", node.getVaList()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } @@ -205,7 +205,7 @@ public void lower(VaListNextArgNode node, LoweringTool tool) { // getStackKind() should be at least int throw VMError.shouldNotReachHereUnexpectedInput(node.getStackKind()); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, node.graph(), tool.getLoweringStage()); args.add("vaList", node.getVaList()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/amd64/PosixAMD64VaListSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/amd64/PosixAMD64VaListSnippets.java index bd724e12ace6..874fb15dfab7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/amd64/PosixAMD64VaListSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/amd64/PosixAMD64VaListSnippets.java @@ -181,7 +181,7 @@ protected class VaListInitializationSnippetsLowering implements NodeLoweringProv @Override public void lower(VaListInitializationNode node, LoweringTool tool) { - Arguments args = new Arguments(vaListInitialization, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(vaListInitialization, node.graph(), tool.getLoweringStage()); args.add("vaList", node.getVaList()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } @@ -210,7 +210,7 @@ public void lower(VaListNextArgNode node, LoweringTool tool) { // getStackKind() should be at least int throw VMError.shouldNotReachHereUnexpectedInput(node.getStackKind()); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, node.graph(), tool.getLoweringStage()); args.add("vaList", node.getVaList()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/riscv64/PosixRISCV64VaListSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/riscv64/PosixRISCV64VaListSnippets.java index 56393f9890da..9505d2b55e5b 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/riscv64/PosixRISCV64VaListSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/riscv64/PosixRISCV64VaListSnippets.java @@ -175,7 +175,7 @@ public void lower(VaListNextArgNode node, LoweringTool tool) { // getStackKind() should be at least int throw VMError.shouldNotReachHereUnexpectedInput(node.getStackKind()); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, node.graph(), tool.getLoweringStage()); args.add("vaListPointer", node.getVaList()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/stackvalue/StackValueSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/stackvalue/StackValueSnippets.java index b71230a3bdad..8ff1e394ef13 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/stackvalue/StackValueSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/stackvalue/StackValueSnippets.java @@ -124,7 +124,7 @@ private void lower(LoweringTool tool, AbstractStateSplit node, int sizeInBytes, StructuredGraph graph = node.graph(); boolean mustNotAllocate = ImageSingletons.lookup(RestrictHeapAccessCallees.class).mustNotAllocate(graph.method()); - SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(stackValueSnippet, graph.getGuardsStage(), tool.getLoweringStage()); + SnippetTemplate.Arguments args = new SnippetTemplate.Arguments(stackValueSnippet, graph, tool.getLoweringStage()); args.add("sizeInBytes", sizeInBytes); args.add("alignmentInBytes", alignmentInBytes); args.add("slotIdentifier", slotIdentity); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MonitorSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MonitorSnippets.java index 1a368fb59ca8..7337d7236400 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MonitorSnippets.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MonitorSnippets.java @@ -155,7 +155,7 @@ protected void lowerLowTier(AccessMonitorNode node, LoweringTool tool) { } else { throw VMError.shouldNotReachHereUnexpectedInput(node); // ExcludeFromJacocoGeneratedReport } - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(snippet, node.graph(), tool.getLoweringStage()); args.add("obj", node.object()); template(tool, node, args).instantiate(tool.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args); } diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java index 35375ebcf149..d49db5e0f511 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/api/SubstrateThreadLocalHandshakeSnippets.java @@ -89,7 +89,7 @@ class SafepointLowering implements NodeLoweringProvider { public void lower(TruffleSafepointNode node, LoweringTool tool) { if (tool.getLoweringStage() == LoweringTool.StandardLoweringStage.LOW_TIER) { StructuredGraph graph = node.graph(); - Arguments args = new Arguments(pollSnippet, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(pollSnippet, graph, tool.getLoweringStage()); args.add("node", node.location()); SnippetTemplate template = template(tool, node, args); template.instantiate(tool.getMetaAccess(), node, DEFAULT_REPLACER, args); diff --git a/vm/ci/ci_common/libgraal.jsonnet b/vm/ci/ci_common/libgraal.jsonnet index 67b0a7651f7f..8df8beb659cc 100644 --- a/vm/ci/ci_common/libgraal.jsonnet +++ b/vm/ci/ci_common/libgraal.jsonnet @@ -37,6 +37,7 @@ local galahad = import '../../../ci/ci_common/galahad-common.libsonnet'; # enable asserts in the JVM building the image and enable asserts in the resulting native image libgraal_compiler:: self.libgraal_compiler_base(), libgraal_compiler_zgc:: self.libgraal_compiler_base(extra_vm_args=['-XX:+UseZGC']), + libgraal_compiler_shenandoah:: self.libgraal_compiler_base(extra_vm_args=['-XX:+UseShenandoahGC']), # enable economy mode building with the -Ob flag libgraal_compiler_quickbuild:: self.libgraal_compiler_base(quickbuild_args=['-Ob']) + { environment+: { @@ -69,6 +70,7 @@ local galahad = import '../../../ci/ci_common/galahad-common.libsonnet'; # -ea assertions are enough to keep execution time reasonable libgraal_truffle: self.libgraal_truffle_base(), libgraal_truffle_zgc: self.libgraal_truffle_base(extra_vm_args=['-XX:+UseZGC']), + libgraal_truffle_shenandoah: self.libgraal_truffle_base(extra_vm_args=['-XX:+UseShenandoahGC']), # enable economy mode building with the -Ob flag libgraal_truffle_quickbuild: self.libgraal_truffle_base(['-Ob']), @@ -174,6 +176,29 @@ local galahad = import '../../../ci/ci_common/galahad-common.libsonnet'; ] ], + # Builds run on all platforms (platform = JDK + OS + ARCH) + local all_platforms_shenandoah_builds = [ + adjust_windows_version(c.vm_base(os(os_arch), arch(os_arch), 'gate')) + + svm_common(os_arch, jdk) + + vm.custom_vm + + g.make_build(jdk, os_arch, task, extra_tasks=self, suite="vm", + include_common_os_arch=false, + tier2_manifest=tier2s, + tier3_manifest=tier3s, + dailies_manifest=dailies, + weeklies_manifest=weeklies, + monthlies_manifest=monthlies).build + + vm["vm_java_" + jdk] + for jdk in [ + "Latest", + ] + for os_arch in all_os_arches + for task in [ + "libgraal_compiler_shenandoah", + "libgraal_truffle_shenandoah", + ] + ], + # Coverage builds local coverage_jdkLatest_builds = [ c.vm_base(os(os_arch), arch(os_arch), 'gate') + @@ -204,6 +229,7 @@ local galahad = import '../../../ci/ci_common/galahad-common.libsonnet'; local all_builds = all_platforms_builds + all_platforms_zgc_builds + + all_platforms_shenandoah_builds + coverage_jdkLatest_builds, builds: if diff --git a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMAllocationSnippets.java b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMAllocationSnippets.java index 08d5964f25e3..499872125504 100644 --- a/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMAllocationSnippets.java +++ b/web-image/src/com.oracle.svm.hosted.webimage/src/com/oracle/svm/hosted/webimage/wasm/gc/WasmLMAllocationSnippets.java @@ -111,7 +111,7 @@ public void lower(FormatObjectNode node, LoweringTool tool) { if (graph.getGuardsStage().areFrameStatesAtSideEffects()) { return; } - Arguments args = new Arguments(formatObject, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(formatObject, graph, tool.getLoweringStage()); args.add("memory", node.getMemory()); args.add("hub", node.getHub()); args.add("rememberedSet", node.getRememberedSet()); @@ -129,7 +129,7 @@ public void lower(FormatArrayNode node, LoweringTool tool) { if (graph.getGuardsStage().areFrameStatesAtSideEffects()) { return; } - Arguments args = new Arguments(formatArray, graph.getGuardsStage(), tool.getLoweringStage()); + Arguments args = new Arguments(formatArray, graph, tool.getLoweringStage()); args.add("memory", node.getMemory()); args.add("hub", node.getHub()); args.add("length", node.getLength());